diff --git a/CI/ESS/envfiles/config.ess.js b/CI/ESS/envfiles/config.ess.js index 3d44ea18..3f9a2ef6 100644 --- a/CI/ESS/envfiles/config.ess.js +++ b/CI/ESS/envfiles/config.ess.js @@ -14,6 +14,16 @@ module.exports = { username: process.env.SCICHAT_USER || "logbookReader", password: process.env.SCICHAT_PASSWORD || "logrdr", }, + rabbitmq: { + enabled: true, + host: "localhost", + port: 5672, + username: "guest", + password: "guest", + // NOTE: The queue name comes from the duo-message-broker package + queue: "SCICAT_PROPOSAL", + }, + proposalCreationStatusTrigger: process.env.PROPOSAL_CREATION_STATUS_TRIGGER || "ALLOCATED", datasetReductionEnabled: false, reductionKafkaBroker: "kafka:9092", reductionKafkaInputTopic: "reduce_input", diff --git a/package-lock.json b/package-lock.json index bb7121d9..70e8ca1a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "BSD-3-Clause", "dependencies": { "@snyk/protect": "^1.936.0", + "@user-office-software/duo-message-broker": "^1.4.0", "amqplib": "^0.9.0", "compression": "^1.7.4", "cors": "^2.8.5", @@ -60,6 +61,45 @@ "loopback-connector-kafka": "^0.2.1" } }, + "node_modules/@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/@acuminous/bitsyntax/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/@babel/code-frame": { "version": "7.12.13", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.13.tgz", @@ -240,6 +280,20 @@ "node": ">=10" } }, + "node_modules/@types/amqplib": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.8.2.tgz", + "integrity": "sha512-p+TFLzo52f8UanB+Nq6gyUi65yecAcRY3nYowU6MPGFtaJvEDxcnFWrxssSTkF+ts1W3zyQDvgVICLQem5WxRA==", + "dependencies": { + "@types/bluebird": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bluebird": { + "version": "3.5.36", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.36.tgz", + "integrity": "sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q==" + }, "node_modules/@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -345,6 +399,43 @@ "@types/node": "*" } }, + "node_modules/@user-office-software/duo-logger": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@user-office-software/duo-logger/-/duo-logger-1.2.0.tgz", + "integrity": "sha512-/4DnoO8ehMIofExIdRrmwE7pOp40KG3PPvgV4dDDGHKz9dlmAvjARAgo46y5etw6L9ME6ZUsEkczFll0D6AQJw==", + "dependencies": { + "fast-safe-stringify": "^2.1.1", + "gelf-pro": "^1.3.6" + } + }, + "node_modules/@user-office-software/duo-message-broker": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@user-office-software/duo-message-broker/-/duo-message-broker-1.4.0.tgz", + "integrity": "sha512-/NUaIA7Ztq0TvbZczpSXJ5++8bOq2ftrdZ/ihmDgzt3Sg7OTwtIcTemOBIO78wHJ4ocyqnAMb3JecouijAjzUA==", + "dependencies": { + "@types/amqplib": "^0.8.2", + "@user-office-software/duo-logger": "^1.1.3", + "amqplib": "^0.10.2" + }, + "engines": { + "node": ">=16.14.0", + "npm": ">=8.5.0" + } + }, + "node_modules/@user-office-software/duo-message-broker/node_modules/amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "dependencies": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -419,9 +510,9 @@ } }, "node_modules/amqplib": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.9.0.tgz", - "integrity": "sha512-emwSdJElmSp52JIKehjLNimKqbZcGUBGdcqST9fll+C/Uss8fWoGyyWlwt20f5lD+SDdozoc4WhF3uDCUOL2ww==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.9.1.tgz", + "integrity": "sha512-a1DP0H1LcLSMKPAnhUN2AKbVyEPqEUrUf7O+odhKGxaO+Tf0nWtuD7Zq5P9uZwZteu56OfW9EQozSCTKsAEk5w==", "dependencies": { "bitsyntax": "~0.1.0", "bluebird": "^3.7.2", @@ -2372,9 +2463,9 @@ "dev": true }, "node_modules/fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, "node_modules/feature-policy": { "version": "0.3.0", @@ -8616,6 +8707,36 @@ } }, "dependencies": { + "@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "requires": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } + } + }, "@babel/code-frame": { "version": "7.12.13", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.13.tgz", @@ -8769,6 +8890,20 @@ "resolved": "https://registry.npmjs.org/@snyk/protect/-/protect-1.936.0.tgz", "integrity": "sha512-gH2JsMQGa/PJBr6ekRQi5IVUh6qFwhEI4yffEYQKiMNQvXaFJidbfyvJnGvRjEKO7ZuM3EnyvWErHXdnvm7Fhw==" }, + "@types/amqplib": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@types/amqplib/-/amqplib-0.8.2.tgz", + "integrity": "sha512-p+TFLzo52f8UanB+Nq6gyUi65yecAcRY3nYowU6MPGFtaJvEDxcnFWrxssSTkF+ts1W3zyQDvgVICLQem5WxRA==", + "requires": { + "@types/bluebird": "*", + "@types/node": "*" + } + }, + "@types/bluebird": { + "version": "3.5.36", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.36.tgz", + "integrity": "sha512-HBNx4lhkxN7bx6P0++W8E289foSu8kO8GCk2unhuVggO+cE7rh9DhZUyPhUxNRG9m+5B5BTKxZQ5ZP92x/mx9Q==" + }, "@types/body-parser": { "version": "1.19.0", "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.0.tgz", @@ -8874,6 +9009,38 @@ "@types/node": "*" } }, + "@user-office-software/duo-logger": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@user-office-software/duo-logger/-/duo-logger-1.2.0.tgz", + "integrity": "sha512-/4DnoO8ehMIofExIdRrmwE7pOp40KG3PPvgV4dDDGHKz9dlmAvjARAgo46y5etw6L9ME6ZUsEkczFll0D6AQJw==", + "requires": { + "fast-safe-stringify": "^2.1.1", + "gelf-pro": "^1.3.6" + } + }, + "@user-office-software/duo-message-broker": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@user-office-software/duo-message-broker/-/duo-message-broker-1.4.0.tgz", + "integrity": "sha512-/NUaIA7Ztq0TvbZczpSXJ5++8bOq2ftrdZ/ihmDgzt3Sg7OTwtIcTemOBIO78wHJ4ocyqnAMb3JecouijAjzUA==", + "requires": { + "@types/amqplib": "^0.8.2", + "@user-office-software/duo-logger": "^1.1.3", + "amqplib": "^0.10.2" + }, + "dependencies": { + "amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "requires": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + } + } + } + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -8931,9 +9098,9 @@ } }, "amqplib": { - "version": "0.9.0", - "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.9.0.tgz", - "integrity": "sha512-emwSdJElmSp52JIKehjLNimKqbZcGUBGdcqST9fll+C/Uss8fWoGyyWlwt20f5lD+SDdozoc4WhF3uDCUOL2ww==", + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.9.1.tgz", + "integrity": "sha512-a1DP0H1LcLSMKPAnhUN2AKbVyEPqEUrUf7O+odhKGxaO+Tf0nWtuD7Zq5P9uZwZteu56OfW9EQozSCTKsAEk5w==", "requires": { "bitsyntax": "~0.1.0", "bluebird": "^3.7.2", @@ -10543,9 +10710,9 @@ "dev": true }, "fast-safe-stringify": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", - "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", + "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" }, "feature-policy": { "version": "0.3.0", diff --git a/package.json b/package.json index 57f93563..d26e52d6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@snyk/protect": "^1.936.0", + "@user-office-software/duo-message-broker": "^1.4.0", "amqplib": "^0.9.0", "compression": "^1.7.4", "cors": "^2.8.5", diff --git a/server/boot/startRabbitMqConsumer.js b/server/boot/startRabbitMqConsumer.js index e537994a..0ba3434c 100644 --- a/server/boot/startRabbitMqConsumer.js +++ b/server/boot/startRabbitMqConsumer.js @@ -1,32 +1,24 @@ "use strict"; -const amqp = require("amqplib"); +const { RabbitMQMessageBroker } = require("@user-office-software/duo-message-broker"); const config = require("../config.local"); const logger = require("../../common/logger"); -const utils = require("../../common/models/utils"); module.exports = async function (app) { const Proposal = app.models.Proposal; let connectionDetails; - const startConsumer = async (connection) => { + const startConsumer = async (rabbitMq) => { try { - const channel = await connection.createChannel(); - await channel.assertExchange("useroffice.fanout", "fanout", { durable: true }); - const queueName = await channel.assertQueue("", { exclusive: true }); - await channel.bindQueue(queueName.queue, "useroffice.fanout", ""); - channel.prefetch(1); - - logger.logInfo("RABBITMQ Waiting for messages", { - queue: queueName.queue - }); - - await channel.consume(queueName.queue, async (msg) => { + await rabbitMq.listenOn(config.rabbitmq.queue, async (type, message) => { try { - const payload = JSON.parse(msg.content); - logger.logInfo("Message properties", JSON.stringify(msg.properties).toString()); - logger.logInfo("Message body", JSON.stringify(payload).toString()); - switch (msg.properties.type) { - case "PROPOSAL_ACCEPTED": { + switch (type) { + case "PROPOSAL_STATUS_CHANGED_BY_WORKFLOW": + case "PROPOSAL_STATUS_CHANGED_BY_USER": { + + // If the status is different then the trigger status then skip the proposal creation + if (message.newStatus !== config.proposalCreationStatusTrigger) { + return; + } /* from useroffice code, courtesy of Jekabs msgJSON @@ -46,28 +38,29 @@ module.exports = async function (app) { lastName email } + newStatus properties type -> event type */ logger.logInfo( - "RabbitMq message for PROPOSAL_ACCEPTED", + "RabbitMq message for " + type, { - message: payload + message: message } ); /* - We need to refactor proposal fields to match scicat - */ + We need to refactor proposal fields to match scicat + */ let proposalData = { - "proposalId": payload.shortCode, - "title": payload.title, - "pi_email": payload.proposer.email, - "pi_firstname": payload.proposer.firstName, - "pi_lastname": payload.proposer.lastName, - "email": payload.proposer.email, - "firstname": payload.proposer.firstName, - "lastname": payload.proposer.lastName, - "abstract": payload.asbtract, + "proposalId": message.shortCode, + "title": message.title, + "pi_email": message.proposer.email, + "pi_firstname": message.proposer.firstName, + "pi_lastname": message.proposer.lastName, + "email": message.proposer.email, + "firstname": message.proposer.firstName, + "lastname": message.proposer.lastName, + "abstract": message.asbtract, "ownerGroup": "ess", "createdBy": "proposalIngestor" }; @@ -91,11 +84,9 @@ module.exports = async function (app) { } } }); - channel.ack(msg); break; } default: { - channel.ack(msg); break; } } @@ -111,78 +102,30 @@ module.exports = async function (app) { } catch (error) { logger.logError(error.message, { location: "startConsumer", - connection - }); - } - }; - - const sendNotificationEmail = () => { - if ("smtpMessage" in config && "to" in config.smtpMessage && config.smtpMessage.to) { - const subjectText = "Failed to connect to RabbitMQ"; - let mailText = "Hello,\n Scicat backend failed to connect to RabbitMQ\n"; - mailText += "Connection details:\n"; - mailText += JSON.stringify(connectionDetails, null, 3); - utils.sendMail(config.smtpMessage.to, "", subjectText, mailText, null, null); - } else { - logger.logWarn("smtpMessage is not configured properly no email was sent", { - location: "sendNotificationEmail" + rabbitMq }); } }; - let connectionAttempts = 0; const connect = async () => { - await amqp.connect(connectionDetails, async function (error, connection) { - if (error) { - logger.logError(error.message, { - location: "amqp.connect", - connectionDetails - }); - if (connectionAttempts < config.rabbitmq.maxReconnectionAttempts) { - console.log("RABBITMQ - Connection attempt " + connectionAttempts); - connectionAttempts++; - // try to connect again after some amount of time - return setTimeout(connect, config.rabbitmq.reconnectionInterval, connectionDetails); - } - console.log("RABBITMQ - Unable to connect"); - sendNotificationEmail(); - return; - } - connection.on("error", (error) => { - logger.logError(error.message, { - location: "connection.on error", - }); - }); - connection.on("close", () => { - logger.logError("RABBITMQ - Reconnecting", { - location: "connection.on close", - }); - if (connectionAttempts < config.rabbitmq.maxReconnectionAttempts) { - console.log("RABBITMQ - Connection attempt " + connectionAttempts); - connectionAttempts++; - // try to connect again after some amount of time - return setTimeout(connect, config.rabbitmq.reconnectionInterval, connectionDetails); - } - console.log("RABBITMQ - Unable to reconnect"); - sendNotificationEmail(); - return; - }); - console.log("RABBITMQ - Connected"); - connectionAttempts = 0; - await startConsumer(connection); + const rabbitMq = new RabbitMQMessageBroker(); + + await rabbitMq.setup({ + hostname: connectionDetails.hostname, + username: connectionDetails.username, + password: connectionDetails.password, }); + + await startConsumer(rabbitMq); }; const rabbitMqEnabled = config.rabbitmq ? config.rabbitmq.enabled : false; if (rabbitMqEnabled) { if (config.rabbitmq.host) { connectionDetails = { - protocol: "amqp", hostname: config.rabbitmq.host, username: config.rabbitmq.username, password: config.rabbitmq.password, - heartbeat: 60, - vhost: ("vhost" in config.rabbitmq) ? config.rabbitmq.vhost : "/", }; if (config.rabbitmq.port) { connectionDetails = { ...connectionDetails, port: config.rabbitmq.port };