Merge Prod Release into Master

.github/workflows/ci.yml

@@ -7,6 +7,6 @@ jobs:
   security:
     uses: dvsa/.github/.github/workflows/nodejs-security.yaml@v2
     with:
-      args: "--all-projects"
+      args: "--all-projects --severity-threshold=high"
     secrets:
       SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

package.json

@@ -31,19 +31,27 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@dvsa/cvs-microservice-common": "^0.9.6",
-    "aws-sdk": "^2.1383.0",
+    "@aws-sdk/client-s3": "^3.540.0",
+    "@aws-sdk/client-secrets-manager": "^3.540.0",
+    "@aws-sdk/lib-storage": "^3.540.0",
+    "@aws-sdk/types": "^3.535.0",
+    "@aws-sdk/util-dynamodb": "^3.609.0",
+    "@dvsa/cvs-microservice-common": "^1.1.0",
+    "@dvsa/cvs-type-definitions": "7.2.0",
+    "@smithy/util-utf8": "^2.3.0",
+    "aws-sdk-client-mock": "^4.0.0",
     "aws-xray-sdk": "^3.3.4",
     "exceljs": "^4.3.0",
     "js-yaml": "^3.13.1",
     "moment": "^2.29.4",
     "moment-timezone": "^0.5.27",
     "node-yaml": "^4.0.1",
     "request": "^2.88.2",
-    "request-promise": "^4.2.5"
+    "request-promise": "^4.2.6",
+    "serverless-dependency-invoke": "file:./packages/serverless-dependency-invoke"
   },
   "devDependencies": {
-    "@commitlint/cli": "^13.1.0",
+    "@commitlint/cli": "^19.2.1",
     "@commitlint/config-conventional": "^13.1.0",
     "@types/aws-lambda": "^8.10.40",
     "@types/chai": "^4.2.7",
@@ -54,21 +62,22 @@
     "@types/node": "10",
     "@types/request-promise": "^4.1.45",
     "@types/sinon": "^7.5.1",
+    "@types/lodash": "4.14.202",
     "audit-filter": "^0.5.0",
     "aws-lambda-mock-context": "^3.2.1",
     "chai": "^4.2.0",
-    "gulp": "^4.0.2",
+    "gulp": "^5.0.0",
     "husky": "^4.2.0",
-    "jest": "^24.9.0",
+    "jest": "^29.7.0",
     "jest-plugin-context": "^2.9.0",
     "prettier": "^2.3.2",
-    "serverless": "^2.45.2",
-    "serverless-offline": "^5.12.1",
+    "serverless": "^3.0.0",
+    "serverless-offline": "^13.0.4",
     "serverless-plugin-tracing": "^2.0.0",
-    "serverless-plugin-typescript": "^1.1.9",
+    "serverless-plugin-typescript": "^2.1.5",
     "sinon": "^8.1.1",
     "sonar-scanner": "^3.1.0",
-    "ts-jest": "^24.3.0",
+    "ts-jest": "^29.1.2",
     "tslint": "^5.20.1",
     "typescript": "4.4.4"
   },

packages/serverless-dependency-invoke/index.js

@@ -0,0 +1,184 @@
+"use strict";
+
+const packagePath = "node_modules/serverless-dependency-invoke";
+const handlerPath = `proxy.js`;
+const exec = require("child_process").exec;
+const path = require("path");
+const fs = require("fs");
+
+class ServerlessPlugin {
+  constructor(serverless, options) {
+    this.serverless = serverless;
+    this.options = options;
+
+    this.hooks = {
+      "before:offline:start:init": this.aquireDependencies.bind(this),
+    };
+  }
+
+  async aquireDependencies() {
+    const config = this.serverless.service.custom["dependency-invoke"];
+    const dependencies = config.dependencies;
+    const storage = path.join(
+      this.serverless.service.serverless.config.servicePath,
+      "dependencies"
+    );
+    let dependencyPromiseArray;
+
+    if (!fs.existsSync(storage)) {
+      fs.mkdirSync(storage);
+    }
+
+    dependencyPromiseArray = await dependencies.map((dependency) => {
+      return new Promise((resolve, reject) => {
+        if (dependency.lang === "javascript") {
+          dependency.storage = path
+            .join(storage, dependency.name)
+            .replace("/.build", "");
+        }
+
+        dependency.handler = path.join(dependency.storage, dependency.handler);
+
+        if (!fs.existsSync(dependency.storage)) {
+          this.serverless.cli.log(`Fetching dependency ${dependency.name}...`);
+          exec(
+            `git clone --single-branch --branch ${dependency.branch} ${dependency.git} ${dependency.storage}`,
+            async (error, stdout, stderr) => {
+              if (error) {
+                reject(error);
+              }
+
+              this.serverless.cli.log(`Fetched dependency ${dependency.name}`);
+
+              this.executeFlow(dependency)
+                .then(() => {
+                  resolve(dependency);
+                })
+                .catch((error) => {
+                  reject(error);
+                });
+            }
+          );
+        } else {
+          this.serverless.cli.log(`Updating dependency ${dependency.name}...`);
+          exec(
+            `cd ${dependency.storage} && git reset --hard origin/${dependency.branch} && git pull`,
+            async (error, stdout, stderr) => {
+              if (error) {
+                reject(error);
+              }
+
+              this.serverless.cli.log(`Updated dependency ${dependency.name}`);
+
+              this.executeFlow(dependency)
+                .then(() => {
+                  resolve(dependency);
+                })
+                .catch((error) => {
+                  reject(error);
+                });
+            }
+          );
+        }
+      });
+    });
+
+    return Promise.all(dependencyPromiseArray).then((dependency) => {
+      this.startHandler(dependencies);
+    });
+  }
+
+  async executeFlow(dependency) {
+    const commands = dependency.commands;
+
+    if (!commands) {
+      throw new Error(
+        `Commands were not defined for dependency ${dependency.name}`
+      );
+    }
+
+    for (const command of commands) {
+      this.serverless.cli.log(`[${dependency.name}] Executing "${command}"...`);
+      await execPromise(`cd ${dependency.storage} && ${command}`).catch(
+        (error) => {
+          this.serverless.cli.log(`Command ${command} failed\n${error}`);
+        }
+      );
+    }
+  }
+
+  startHandler(dependencies) {
+    let location = "";
+    try {
+      location = this.serverless.service.custom["serverless-offline"].location;
+      this.serverless.service.custom["serverless-offline"].location = "";
+    } catch (_) {}
+
+    const storage = path.join(
+      this.serverless.service.serverless.config.servicePath,
+      "dependencies"
+    );
+
+    const functions = this.serverless.service.functions;
+
+    dependencies.forEach((dependency) => {
+      this.serverless.cli.log(`Dependency ${dependency.name} loaded`);
+      functions[dependency.name] = functionProxy(dependency, location);
+    });
+  }
+}
+
+const functionProxy = (dependency, location) => ({
+  name: `${dependency.name}_proxy`,
+  handler: `${packagePath}/proxy.handler`,
+  events: [
+    {
+      http: {
+        method: "POST",
+        path: `2015-03-31/functions/${dependency.name}/invocations`,
+        integration: "lambda",
+        request: {
+          template: {
+            "binary/octet-stream": JSON.stringify({
+              location,
+              body: "$input.body",
+              headers: {
+                "X-Amz-Invocation-Type":
+                  "$input.params().header.get('X-Amz-Invocation-Type')",
+                "X-Amz-Log-Type":
+                  "$input.params().header.get('X-Amz-Log-Type')",
+                "X-Amz-Client-Context":
+                  "$input.params().header.get('X-Amz-Client-Context')",
+              },
+              targetHandler: dependency.handler,
+            }),
+          },
+        },
+        response: {
+          headers: {
+            "Content-Type": "application/json",
+          },
+        },
+      },
+    },
+  ],
+  package: {
+    include: [handlerPath],
+  },
+});
+
+const execPromise = (command) => {
+  return new Promise((resolve, reject) => {
+    const childProcess = exec(command, (error, stdout, stderr) => {
+      if (error || stderr) reject(error);
+      resolve(stdout);
+    });
+
+    childProcess.on("exit", function (code) {
+      if (code === 0) resolve(code);
+      else reject(code);
+    });
+  });
+};
+
+module.exports = ServerlessPlugin;

packages/serverless-dependency-invoke/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "serverless-dependency-invoke",
+  "version": "0.0.10",
+  "description": "A Serverless plugin that allows you to declare dependency projects from git repositories which can be invoked using aws-sdk.",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/gmlazu/serverless-dependency-invoke"
+  },
+  "bugs": {
+    "url": "https://github.com/gmlazu/serverless-dependency-invoke/issues"
+  },
+  "homepage": "https://github.com/gmlazu/serverless-dependency-invoke",
+  "keywords": [
+    "AWS",
+    "Lambda",
+    "Serverless",
+    "Offline",
+    "Dependency",
+    "Injection"
+  ],
+  "author": "Lazu George Mihai",
+  "license": "MIT"
+}

packages/serverless-dependency-invoke/proxy.js

@@ -0,0 +1,43 @@
+const path = require("path");
+
+function handler(event, context, callback) {
+  // extract the path to the handler (relative to the project root)
+  // and the function to call on the handler
+  const [targetHandlerFile, targetHandlerFunction] =
+    event.targetHandler.split(".");
+  const target = require(path.resolve(
+    __dirname,
+    "../..",
+    event.location,
+    targetHandlerFile
+  ));
+  const headers = event.headers;
+
+  if (headers["X-Amz-Invocation-Type"] === "DryRun") {
+    return;
+  }
+
+  // call the target function
+  const targetResponse = target[targetHandlerFunction](
+    event.body,
+    context,
+    (error, response) => {
+      if (headers["X-Amz-Invocation-Type"] === "RequestResponse") {
+        if (error) {
+          callback(error);
+        }
+
+        callback(null, JSON.stringify(response));
+      }
+    }
+  );
+
+  if (
+    targetResponse &&
+    headers["X-Amz-Invocation-Type"] === "RequestResponse"
+  ) {
+    return targetResponse;
+  }
+}
+
+module.exports.handler = handler;

src/functions/retroGen.ts

@@ -1,8 +1,9 @@
-import { processRecord } from "@dvsa/cvs-microservice-common/functions/sqsFilter";
-import { Lambda } from "aws-sdk";
-import { ManagedUpload } from "aws-sdk/clients/s3";
+import { LambdaClient } from "@aws-sdk/client-lambda";
+import { unmarshall } from "@aws-sdk/util-dynamodb";
+import { SQSBatchItemFailure, SQSBatchResponse } from "aws-lambda";
 import * as rp from "request-promise";
 import { ERRORS } from "../assets/Enum";
+import { credentials } from "../handler";
 import { ActivitiesService } from "../services/ActivitiesService";
 import { LambdaService } from "../services/LambdaService";
 import { RetroGenerationService } from "../services/RetroGenerationService";
@@ -12,39 +13,36 @@ import { TestResultsService } from "../services/TestResultsService";
 
 /**
  * λ function to process a DynamoDB stream of test results into a queue for certificate generation.
- * @param event - DynamoDB Stream event
- * @param context - λ Context
- * @param callback - callback function
+ * @param event - SQS event that contains a DynamoDB stream event to parse out
  */
-const retroGen = async (event: any): Promise<void | ManagedUpload.SendData[]> => {
+const retroGen = async (event: any): Promise<SQSBatchResponse> => {
   if (!event || !event.Records || !Array.isArray(event.Records) || !event.Records.length) {
     console.error("ERROR: event is not defined.");
     throw new Error(ERRORS.EventIsEmpty);
   }
-  const retroService: RetroGenerationService = new RetroGenerationService(new TestResultsService(new LambdaService(new Lambda())), new ActivitiesService(new LambdaService(new Lambda())));
-  const retroUploadPromises: Array<Promise<ManagedUpload.SendData>> = [];
+
+  const retroService: RetroGenerationService = new RetroGenerationService(new TestResultsService(new LambdaService(new LambdaClient({ ...credentials }))), new ActivitiesService(new LambdaService(new LambdaClient({ ...credentials }))));
   const sharepointAuthenticationService = new SharePointAuthenticationService(rp);
   const sharePointService = new SharePointService(rp);
+  const batchItemFailures: SQSBatchItemFailure[] = [];
 
-  event.Records.forEach((record: any) => {
-    const recordBody = JSON.parse(JSON.parse(record.body).Message);
-    const visit: any = processRecord(recordBody);
-    if (visit) {
-      const retroUploadPromise = retroService.generateRetroReport(visit).then(async (generationServiceResponse: { fileName: string; fileBuffer: Buffer }) => {
+  for (const record of event.Records) {
+    try {
+      const recordBody = JSON.parse(record.body);
+      const visit: any = unmarshall(recordBody?.dynamodb.NewImage);
+      if (visit) {
+        const generationServiceResponse = await retroService.generateRetroReport(visit);
         const tokenResponse = await sharepointAuthenticationService.getToken();
         const accessToken = JSON.parse(tokenResponse).access_token;
-        const sharePointResponse = await sharePointService.upload(generationServiceResponse.fileName, generationServiceResponse.fileBuffer, accessToken);
-        return sharePointResponse;
-      });
-
-      retroUploadPromises.push(retroUploadPromise);
+        await sharePointService.upload(generationServiceResponse.fileName, generationServiceResponse.fileBuffer, accessToken);
+      }
+    } catch (error) {
+      console.error(`record id ${record.messageId}, error: ${error}`);
+      batchItemFailures.push({ itemIdentifier: record.messageId });
     }
-  });
+  }
 
-  return Promise.all(retroUploadPromises).catch((error: any) => {
-    console.error(error);
-    throw error;
-  });
+  return { batchItemFailures };
 };
 
 export { retroGen };