feat(CB2-12965): Added additional logs (#114)

* feat(CB2-12695): Added addiitonal logs * feat(CB2-12695): fixed linting issue * feat(CB2-12695): fixed linting issue * feat(CB2-12965): fixed linting issue * feat(CB2-12965): added semi colons * feat(CB2-12965): added semi colons * feat(CB2-12965): unit test to cover console log * feat(CB2-12965): unit test to cover console log * feat(CB2-12965): change let ot const * feat(CB2-12965): Added operation type to the log * feat(CB2-12965): Formatting in logger unit test
dvsa · Jul 15, 2024 · 56c20f6 · 56c20f6
1 parent 2cb4d81
commit 56c20f6
Show file tree

Hide file tree

Showing 7 changed files with 308 additions and 87 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -55,6 +55,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@dvsa/cvs-type-definitions": "^7.2.0",
     "@aws-sdk/client-dynamodb": "3.577.0",
     "@aws-sdk/client-dynamodb-streams": "3.577.0",
     "@aws-sdk/client-secrets-manager": "3.565.0",

diff --git a/src/functions/process-stream-event.ts b/src/functions/process-stream-event.ts
@@ -14,22 +14,26 @@ import { convert } from '../services/entity-conversion';
 import { debugLog } from '../services/logger';
 import { SqlOperation, deriveSqlOperation } from '../services/sql-operations';
 import { transformTechRecord } from '../utils/transform-tech-record';
+import { unmarshall } from "@aws-sdk/util-dynamodb";
+
+let logManager: ILog[] = [];
 
 /**
  * λ function: convert a DynamoDB document to Aurora RDS rows
  * @param event - DynamoDB stream event, containing DynamoDB document image
  * @param context - λ context
  */
 export const processStreamEvent: Handler = async (
-  event: SQSEvent,
-  context: Context,
+    event: SQSEvent,
+    context: Context,
 ): Promise<any> => {
   const res: BatchItemFailuresResponse = {
     batchItemFailures: [],
   };
-
   try {
+    const processStartTime: Date = new Date();
     debugLog('Received SQS event: ', JSON.stringify(event));
+    let iLog: ILog = { changeType: "", identifier: "", operationType: "" };
 
     validateEvent(event);
 
@@ -52,50 +56,69 @@ export const processStreamEvent: Handler = async (
 
       // parse source ARN
       const tableName: string = getTableNameFromArn(
-        dynamoRecord.eventSourceARN!,
+          dynamoRecord.eventSourceARN!,
       );
 
       if (tableName.includes('flat-tech-records')) {
         transformTechRecord(dynamoRecord as _Record);
         debugLog(`Dynamo Record after transformation: ${dynamoRecord}`);
+        const technicalRecord: any = dynamoRecord.dynamodb?.NewImage;
+        const unmarshalledTechnicalRecord = unmarshall(technicalRecord);
+        iLog.statusCode = unmarshalledTechnicalRecord.statusCode;
+        iLog.changeType = "Technical Record Change";
+        iLog.identifier = unmarshalledTechnicalRecord.vehicleType === 'trl'
+            ? unmarshalledTechnicalRecord.trailerId
+            : unmarshalledTechnicalRecord.primaryVrm;
+      } else if (tableName.includes('test-results')) {
+        const testResult: any = dynamoRecord.dynamodb?.NewImage;
+        const unmarshalledTestResult = unmarshall(testResult);
+        iLog.changeType = 'Test record change';
+        iLog.testResultId = unmarshalledTestResult.testResultId;
+        iLog.identifier = unmarshalledTestResult.vehicleType === 'trl'
+            ? unmarshalledTestResult.trailerId :
+            unmarshalledTestResult.primaryVrm;
       }
-
       // is this an INSERT, UPDATE, or DELETE?
       const operationType: SqlOperation = deriveSqlOperation(
-        dynamoRecord.eventName!,
+          dynamoRecord.eventName!,
       );
 
+      iLog.operationType = operationType;
+      addToILog(iLog);
+
       // parse native DynamoDB format to usable TS map
       const image: DynamoDbImage = selectImage(
-        operationType,
-        dynamoRecord.dynamodb!,
+          operationType,
+          dynamoRecord.dynamodb!,
       );
 
       debugLog('Dynamo image dump:', image);
 
       try {
         debugLog(
-          `DynamoDB ---> Aurora | START (event ID: ${dynamoRecord.eventID})`,
+            `DynamoDB ---> Aurora | START (event ID: ${dynamoRecord.eventID})`,
         );
 
         await convert(tableName, operationType, image);
 
         debugLog(
-          `DynamoDB ---> Aurora | END   (event ID: ${dynamoRecord.eventID})`,
+            `DynamoDB ---> Aurora | END   (event ID: ${dynamoRecord.eventID})`,
         );
+        console.log(`** RESULTS **\nProcess start time is: ${processStartTime.toISOString()} \n${JSON.stringify(logManager)}`,
+        )
       } catch (err) {
         console.error(
-          "Couldn't convert DynamoDB entity to Aurora, will return record to SQS for retry",
-          [`messageId: ${id}`, err],
+            "Couldn't convert DynamoDB entity to Aurora, will return record to SQS for retry",
+            [`messageId: ${id}`, err],
         );
-        res.batchItemFailures.push({ itemIdentifier: id });
+        res.batchItemFailures.push({itemIdentifier: id});
         dumpArguments(event, context);
       }
     }
   } catch (err) {
     console.error(
-      'An error unrelated to Dynamo-to-Aurora conversion has occurred, event will not be retried',
-      err,
+        'An error unrelated to Dynamo-to-Aurora conversion has occurred, event will not be retried',
+        err,
     );
     dumpArguments(event, context);
     await destroyConnectionPool();
@@ -107,9 +130,9 @@ export const processStreamEvent: Handler = async (
 export const getTableNameFromArn = (eventSourceArn: string): string => eventSourceArn.split(':')[5].split('/')[1];
 
 const selectImage = (
-  operationType: SqlOperation,
-  streamRecord: StreamRecord,
-  // eslint-disable-next-line consistent-return
+    operationType: SqlOperation,
+    streamRecord: StreamRecord,
+    // eslint-disable-next-line consistent-return
 ): DynamoDbImage => {
   // eslint-disable-next-line default-case
   switch (operationType) {
@@ -165,3 +188,7 @@ const dumpArguments = (event: DynamoDBStreamEvent, context: Context): void => {
   console.error('Event dump  : ', JSON.stringify(event));
   console.error('Context dump: ', JSON.stringify(context));
 };
+
+const addToILog = (iLog: ILog) => {
+  if (iLog.identifier && iLog.changeType) logManager.push(iLog);
+};
diff --git a/src/models/ILog.ts b/src/models/ILog.ts
@@ -0,0 +1,7 @@
+interface ILog {
+    changeType: string;
+    identifier: string;
+    operationType: string;
+    statusCode?: string;
+    testResultId?: string;
+}
diff --git a/src/utils/transform-tech-record.ts b/src/utils/transform-tech-record.ts
@@ -36,7 +36,7 @@ const nestItem = (record: LegacyKeyStructure, key: string, value: string | numbe
   return record;
 };
 
-const transformImage = (image: NewKeyStructure) => {
+export const transformImage = (image: NewKeyStructure) => {
   const vehicle = {} as LegacyTechRecord;
   vehicle.techRecord = [];
 

diff --git a/tests/unit/functions/process-stream-event.unitTest.ts b/tests/unit/functions/process-stream-event.unitTest.ts
@@ -5,6 +5,9 @@ import {
 } from '../../../src/functions/process-stream-event';
 import { convert } from '../../../src/services/entity-conversion';
 import { exampleContext } from '../../utils';
+import testResultWithTestType from '../../resources/dynamodb-image-test-results-with-testtypes.json';
+import techRecordV3 from '../../resources/dynamodb-image-technical-record-V3.json';
+
 
 jest.mock('../../../src/services/entity-conversion', () => ({
   convert: jest.fn(),
@@ -16,7 +19,7 @@ describe('processStreamEvent()', () => {
     mocked(convert).mockResolvedValueOnce({});
   });
 
-  it('should allow valid events to reach the entity conversion procedure', async () => {
+  it('should allow valid events to reach the entity conversion procedure TECHNICAL RECORD', async () => {
     await expect(
       processStreamEvent(
         {
@@ -25,10 +28,10 @@ describe('processStreamEvent()', () => {
               body: JSON.stringify({
                 eventName: 'INSERT',
                 dynamodb: {
-                  NewImage: {},
+                  NewImage: techRecordV3
                 },
                 eventSourceARN:
-                  'arn:aws:dynamodb:eu-west-1:1:table/technical-records/stream/2020-01-01T00:00:00.000',
+                  'arn:aws:dynamodb:eu-west-1:1:table/flat-tech-records/stream/2020-01-01T00:00:00.000',
               }),
             },
           ],
@@ -42,6 +45,60 @@ describe('processStreamEvent()', () => {
     expect(convert).toHaveBeenCalledTimes(1);
   });
 
+  it('should allow valid events to reach the entity conversion procedure test RECORD TRL', async () => {
+    await expect(
+        processStreamEvent(
+            {
+              Records: [
+                {
+                  body: JSON.stringify({
+                    eventName: 'INSERT',
+                    dynamodb: {
+                      NewImage: testResultWithTestType,
+                    },
+                    eventSourceARN:
+                        'arn:aws:dynamodb:eu-west-1:1:table/test-result/stream/2020-01-01T00:00:00.000',
+                  }),
+                },
+              ],
+            },
+            exampleContext(),
+            () => {
+
+            },
+        ),
+    ).resolves.not.toThrow();
+    expect(convert).toHaveBeenCalledTimes(1);
+  });
+
+  it('should allow valid events to reach the entity conversion procedure test RECORD TRL and produce result log', async () => {
+    const consoleSpy = jest.spyOn(console, 'log');
+
+    await expect(
+        processStreamEvent(
+            {
+              Records: [
+                {
+                  body: JSON.stringify({
+                    eventName: 'INSERT',
+                    dynamodb: {
+                      NewImage: testResultWithTestType,
+                    },
+                    eventSourceARN:
+                        'arn:aws:dynamodb:eu-west-1:1:table/test-result/stream/2020-01-01T00:00:00.000',
+                  }),
+                },
+              ],
+            },
+            exampleContext(),
+            () => {
+            },
+        ),
+    ).resolves.not.toThrow();
+    expect(convert).toHaveBeenCalledTimes(1);
+    expect(consoleSpy).toHaveBeenCalledWith(expect.stringContaining('[{\"changeType\":\"Technical Record Change\",\"identifier\":\"VRM-1\",\"operationType\":\"INSERT\"}]'));
+  });
+
   it('should fail on null event', async () => {
     await expect(
       processStreamEvent(null, exampleContext(), () => {

diff --git a/tests/unit/services/logger.unitTest.ts b/tests/unit/services/logger.unitTest.ts
@@ -0,0 +1,10 @@
+import { debugLog } from "../../../src/services/logger";
+
+describe('logger service', () => {
+    it('Should be able to get a logger instance', () => {
+        process.env.DEBUG = 'true'
+        let consoleSpy = jest.spyOn(console, 'info');
+        debugLog("Test");
+        expect(consoleSpy).toHaveBeenCalledWith('Test', []);
+    })
+})