From 279bac0f17cbb7e17ff9b4af6be4d07f34394003 Mon Sep 17 00:00:00 2001 From: imranalisyed506 <105209301+imranalisyed506@users.noreply.github.com> Date: Fri, 5 Jul 2024 16:23:13 +0530 Subject: [PATCH] [PAWS][ENG-55567] Replace GCP collector npm library google-cloud-logging to googleapis to convert protopayload to json (#372) * [PAWS][ENG-55567] Replace GCP collector npm library google-cloud-logging to googleapis to convert protopayload to json --- collectors/googlestackdriver/collector.js | 106 +++-- .../local/events/event_poll.json | 36 +- collectors/googlestackdriver/local/run-sam.sh | 14 + .../googlestackdriver/local/sam-template.yaml | 6 +- collectors/googlestackdriver/package.json | 6 +- collectors/googlestackdriver/test/mock.js | 385 +++++++++++++----- collectors/googlestackdriver/test/test.js | 125 +++--- collectors/gsuite/test/gsuite_test.js | 6 +- ps_spec.yml | 2 +- 9 files changed, 438 insertions(+), 248 deletions(-) diff --git a/collectors/googlestackdriver/collector.js b/collectors/googlestackdriver/collector.js index 4da9e799..a641db26 100644 --- a/collectors/googlestackdriver/collector.js +++ b/collectors/googlestackdriver/collector.js @@ -14,35 +14,36 @@ const PawsCollector = require('@alertlogic/paws-collector').PawsCollector; const calcNextCollectionInterval = require('@alertlogic/paws-collector').calcNextCollectionInterval; const parse = require('@alertlogic/al-collector-js').Parse; const AlLogger = require('@alertlogic/al-aws-collector-js').Logger; -const logging = require('@google-cloud/logging'); const packageJson = require('./package.json'); -const protoFiles = require('google-proto-files'); +const { auth } = require("google-auth-library"); +const { google } = require("googleapis"); const API_THROTTLING_ERROR = 8; const API_THROTTLING_STATUS_CODE = 429; const MAX_POLL_INTERVAL = 900; const MAX_PAGE_SIZE = 1000; -const AUDIT_PAYLOAD_TYPE_URL = 'type.googleapis.com/google.cloud.audit.AuditLog'; +const SCOPES = [ + 'https://www.googleapis.com/auth/cloud-platform', + 'https://www.googleapis.com/auth/cloud-platform.read-only', + 'https://www.googleapis.com/auth/logging.admin', + 'https://www.googleapis.com/auth/logging.read', + 'https://www.googleapis.com/auth/logging.write', +]; const typeIdPaths = [ - {path: ['jsonPayload', 'fields', 'event_type', 'stringValue']}, - {path: ['protoPayload', 'type_url']}, + {path: ['jsonPayload']}, + {path: ['protoPayload', '@type']}, {path: ['payload']} ]; +const tsPaths = [{ path: ["timestamp"] }]; + class GooglestackdriverCollector extends PawsCollector { constructor(context, creds){ super(context, creds, packageJson.version); - this._initAuditLogDecoder(); } - _initAuditLogDecoder() { - const protoPath = protoFiles.getProtoPath('cloud', 'audit', 'audit_log.proto'); - const root = protoFiles.loadSync(protoPath); - const auditLogDecoder = root.lookupType('google.cloud.audit.AuditLog'); - this._auditLogDecoder = auditLogDecoder; - } pawsInitCollectionState(event, callback) { const startTs = process.env.paws_collection_start_ts ? @@ -79,9 +80,18 @@ class GooglestackdriverCollector extends PawsCollector { if (!state.stream) { state = collector.setStreamToCollectionState(state); } + const keysEnvVar = collector.secret; + if (!keysEnvVar) { + throw new Error("The $CREDS environment variable was not found!"); + } // Start API client - const client = new logging.v2.LoggingServiceV2Client({ - credentials: JSON.parse(collector.secret) + const keys = JSON.parse(keysEnvVar); + const client = auth.fromJSON(keys); + client.subject = collector.clientId; + client.scopes = SCOPES; + const logging = google.logging({ + version: 'v2', + auth: client, }); @@ -92,49 +102,46 @@ class GooglestackdriverCollector extends PawsCollector { timestamp < "${state.until}"`; let pagesRetireved = 0; - const options = {autoPaginate: false}; - + const paginationCallback = (result, acc = []) => { - AlLogger.info(`Getting page: ${pagesRetireved + 1} Logs retrieved: ${result[0].length}`); + let logs = result.data.entries || []; + AlLogger.info(`Getting page: ${pagesRetireved + 1} Logs retrieved: ${logs.length}`); pagesRetireved++; - //decode the protoPayload if it's an AuditLog message - let logs = result[0].map(function (logEntry) { - return collector.decodeProtoPayload(logEntry); - }); - - const nextPage = result[1]; + const nextPage = { ...params, pageToken: result.data.nextPageToken }; const newAcc = [...acc, ...logs]; AlLogger.info(`Total Logs ${newAcc.length}`); - if(nextPage && pagesRetireved < process.env.paws_max_pages_per_invocation){ + if (nextPage.pageToken && pagesRetireved < process.env.paws_max_pages_per_invocation) { - return client.listLogEntries(nextPage, options) - .then(res => paginationCallback(res, newAcc)); - } else{ - return {logs: newAcc, nextPage}; + return logging.entries.list(params) + .then(res => { + return paginationCallback(res, newAcc) + }); + } else { + return { logs: newAcc, nextPage }; } }; const pageSize = state.pageSize > 0 ? state.pageSize : MAX_PAGE_SIZE; let params = state.nextPage ? - state.nextPage: + state.nextPage : { filter, pageSize: pageSize, - resourceNames:[state.stream] + resourceNames: [state.stream] }; - client.listLogEntries(params, options) + logging.entries.list(params) .then(paginationCallback) - .then(({logs, nextPage}) => { + .then(({ logs, nextPage }) => { + const newState = collector._getNextCollectionState(state, nextPage); AlLogger.info(`GSTA000002 Next collection in ${newState.poll_interval_sec} seconds`); return callback(null, logs, newState, newState.poll_interval_sec); }) .catch(err => { - AlLogger.error(`GSTA000003 err in collection ${JSON.stringify(err.details)}`); - + AlLogger.error(`GSTA000003 err in collection ${JSON.stringify(err)}`); // Stackdriver Logging api has some rate limits that we might run into. // If we run inot a rate limit error, instead of returning the error, // we return the state back to the queue with an additional second added, up to 15 min @@ -147,12 +154,11 @@ timestamp < "${state.until}"`; const interval = state.poll_interval_sec < 60 ? 60 : state.poll_interval_sec; const nextPollInterval = state.poll_interval_sec < MAX_POLL_INTERVAL ? interval + 60 : MAX_POLL_INTERVAL; - - if (state.nextPage && state.nextPage.pageSize) { + if (state.nextPage && state.nextPage.pageToken && state.nextPage.pageSize) { state.nextPage.pageSize = Math.ceil(state.nextPage.pageSize / 2); AlLogger.debug(`Throttling error with nextPage: ${err.message}. Retrying with smaller pageSize.`); } else { - if (currentInterval <= 15 && err.details.includes('Received message larger than max')) { + if (currentInterval <= 15 && err.message && err.message.indexOf('Received message larger than max') >= 0) { state.pageSize = state.pageSize ? Math.ceil(state.pageSize / 2) : Math.ceil(params.pageSize / 2); AlLogger.debug(`Throttling error with no nextPage and large message: ${err.message}. Reducing pageSize.`); } else { @@ -160,13 +166,13 @@ timestamp < "${state.until}"`; AlLogger.debug(`Throttling error with no nextPage: ${err.message}. Reducing time range.`); } } - const backOffState = Object.assign({}, state, {poll_interval_sec:nextPollInterval}); + const backOffState = Object.assign({}, state, { poll_interval_sec: nextPollInterval }); collector.reportApiThrottling(function () { return callback(null, [], backOffState, nextPollInterval); }); } else { - // set errorCode if not available in error object to showcase client error on DDMetrics - if (err.code) { + // set errorCode if not available in error object to showcase client error on DDMetrics + if (err.code) { err.errorCode = err.code; } return callback(err); @@ -174,19 +180,6 @@ timestamp < "${state.until}"`; }); } - decodeProtoPayload(logEntry) { - let collector = this; - if (logEntry.protoPayload && (logEntry.protoPayload.type_url === AUDIT_PAYLOAD_TYPE_URL)) { - try { - const buffer = Buffer.from(logEntry.protoPayload.value); - let decodedData = collector._auditLogDecoder.decode(buffer); - logEntry.protoPayload.value = decodedData.toJSON(); - } catch(error) { - AlLogger.error(`Error decoding data ${error}`); - } - } - return logEntry; - } _getNextCollectionState(curState, nextPage) { // Reset the page size for the next collection if it's less than the maximum @@ -194,7 +187,7 @@ timestamp < "${state.until}"`; const { stream, since, until } = curState; - if (nextPage) { + if (nextPage && nextPage.pageToken) { // Case: Continue with the next page return { since, @@ -222,15 +215,14 @@ timestamp < "${state.until}"`; // TODO: probably need to actually decode hte protobuf payload on these logs pawsFormatLog(msg) { let collector = this; - - const ts = msg.timestamp ? msg.timestamp : {seconds: Date.now() / 1000}; + const ts = parse.getMsgTs(msg, tsPaths); const typeId = parse.getMsgTypeId(msg, typeIdPaths); let formattedMsg = { // TODO: figure out if this TS is always a string or if they API is goofy... hostname: collector.collector_id, - messageTs: parseInt(ts.seconds), + messageTs: ts.sec, priority: 11, progName: 'GooglestackdriverCollector', message: JSON.stringify(msg), diff --git a/collectors/googlestackdriver/local/events/event_poll.json b/collectors/googlestackdriver/local/events/event_poll.json index b8c55a58..02a408bc 100644 --- a/collectors/googlestackdriver/local/events/event_poll.json +++ b/collectors/googlestackdriver/local/events/event_poll.json @@ -1,20 +1,20 @@ { - "Records": [ - { - "messageId": "f67a45fc-ab43-4df1-9f2f-585bb2043122", - "receiptHandle": "AQEB78LmKaaZ+uj8DVnc/O2zQcAe+qKSi3ZGTSBFssIRSmpwwzUEi2vhPTaIBnhOoh0aoRxVjWdoXO3ZloINfMxmcjycP3KC0WXwcWokoOc3iMCdqhYg0NcOhQW1X0ixc79C9/5/XF1xGd79vLhFL7KvRjjiT4sOaSxlAv6v2fJ5eDETnp7CRa5pocCF4EO2su0M4/TnlLreGfsY+C+/tH+r19AM+d3Jt5dbNMrKMWRRZ7/PTjczkIM7U38AHPuusuBz9uzA5yMQGOMI8FPXfqgcafEN17JqKuNSd/l54v1+s9rBQSzL/MH9wZ0XpZditcpe5pTc+dGuRHOXbFK2A0YUQCvRq1Ed8tfr68uELTQK5jWHH/zPnEMWsTyco9VuNCKr", - "body": "{\n \"priv_collector_state\": {\n \"since\": \"2019-01-01T10:00:00Z\",\n \"until\": \"2020-01-01T10:30:00Z\"\n }\n}", - "attributes": { - "ApproximateReceiveCount": "1", - "SentTimestamp": "1574159909907", - "SenderId": "some-id", - "ApproximateFirstReceiveTimestamp": "1574159909968" - }, - "messageAttributes": {}, - "md5OfBody": "1845296051d4cfe2e6175358a065cc38", - "eventSource": "aws:sqs", - "eventSourceARN": "arn:aws:sqs:us-east-1:352283894008:paws-state-queue", - "awsRegion": "us-east-1" - } - ] + "Records": [ + { + "messageId": "1b77ecca-5c1d-4c88-a6b5-f5be6d6e3947", + "receiptHandle": "AQEB78LmKaaZ+uj8DVnc/O2zQcAe+qKSi3ZGTSBFssIRSmpwwzUEi2vhPTaIBnhOoh0aoRxVjWdoXO3ZloINfMxmcjycP3KC0WXwcWokoOc3iMCdqhYg0NcOhQW1X0ixc79C9/5/XF1xGd79vLhFL7KvRjjiT4sOaSxlAv6v2fJ5eDETnp7CRa5pocCF4EO2su0M4/TnlLreGfsY+C+/tH+r19AM+d3Jt5dbNMrKMWRRZ7/PTjczkIM7U38AHPuusuBz9uzA5yMQGOMI8FPXfqgcafEN17JqKuNSd/l54v1+s9rBQSzL/MH9wZ0XpZditcpe5pTc+dGuRHOXbFK2A0YUQCvRq1Ed8tfr68uELTQK5jWHH/zPnEMWsTyco9VuNCKr", + "body": "{\n \"priv_collector_state\": {\n \"since\": \"2024-04-13T00:00:00Z\",\n \n \"stream\": \"projects/rcs-test-project-422212\",\n \"until\": \"2024-06-13T23:00:00Z\"\n }\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1574159909907", + "SenderId": "some-id", + "ApproximateFirstReceiveTimestamp": "1574159909968" + }, + "messageAttributes": {}, + "md5OfBody": "1845296051d4cfe2e6175358a065cc38", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:352283894008:paws-state-queue", + "awsRegion": "us-east-1" + } + ] } diff --git a/collectors/googlestackdriver/local/run-sam.sh b/collectors/googlestackdriver/local/run-sam.sh index 0260f5ad..90e72071 100755 --- a/collectors/googlestackdriver/local/run-sam.sh +++ b/collectors/googlestackdriver/local/run-sam.sh @@ -7,7 +7,20 @@ SRC_SAM_TEMPLATE="${SCRIPT_DIR}/sam-template.yaml" SRC_ENV_FILE="${SCRIPT_DIR}/${ENV_FILE_NAME}" SRC_EVENT_FILE="${SCRIPT_DIR}/events/${EVENT_FILE_NAME}" RUN_DIR=${SCRIPT_DIR}/../ +PROFILE_NAME="" +exists(){ + command -v "$1" >/dev/null 2>&1 +} + +if exists jq; then + uid=`uuidgen` + LOWERUUID=$(echo "$uid" | tr '[:upper:]' '[:lower:]') + echo "generating messageId in event.json: ${LOWERUUID}" + jq --arg newRandomvalue $LOWERUUID '(.Records[].messageId) |= $newRandomvalue' ${SRC_EVENT_FILE} > tmp && mv tmp ${SRC_EVENT_FILE} +else + echo "jq does not exist please install jq to run command" +fi command -v sam > /dev/null if [ $? -ne 0 ]; then @@ -26,6 +39,7 @@ ln -sf ${SRC_ENV_FILE} ${RUN_DIR}/${ENV_FILE_NAME} ln -sf ${SRC_EVENT_FILE} ${RUN_DIR}/${EVENT_FILE_NAME} cd ${RUN_DIR} && \ sam local invoke \ + --profile ${PROFILE_NAME} \ --env-vars ${ENV_FILE_NAME} \ -t ${SAM_TEMPLATE_NAME} \ -e ${EVENT_FILE_NAME} \ diff --git a/collectors/googlestackdriver/local/sam-template.yaml b/collectors/googlestackdriver/local/sam-template.yaml index a4fd82e6..f595468f 100644 --- a/collectors/googlestackdriver/local/sam-template.yaml +++ b/collectors/googlestackdriver/local/sam-template.yaml @@ -6,7 +6,7 @@ Resources: LocalLambda: Type: AWS::Serverless::Function Properties: - KmsKeyArn: arn:aws:kms:us-east-1:352283894008:key/cdda86d5-615b-4dcc-9319-77ab34510473 + KmsKeyArn: Environment: Variables: AWS_LAMBDA_FUNCTION_NAME: @@ -25,6 +25,7 @@ Resources: paws_api_client_id: paws_max_pages_per_invocation: paws_collector_param_string_2: + paws_collector_param_string_1: paws_endpoint: paws_extension: collector_id: @@ -33,8 +34,9 @@ Resources: customer_id: paws_type_name: ssm_direct: + collector_status_api: api.product.dev.alertlogic.com CodeUri: Runtime: nodejs20.x Handler: index.handler - Timeout: 60 + Timeout: 600 MemorySize: 1024 diff --git a/collectors/googlestackdriver/package.json b/collectors/googlestackdriver/package.json index 6a6c9bc3..2ab38d29 100644 --- a/collectors/googlestackdriver/package.json +++ b/collectors/googlestackdriver/package.json @@ -1,6 +1,6 @@ { "name": "googlestackdriver-collector", - "version": "1.2.11", + "version": "1.2.12", "description": "Alert Logic AWS based Googlestackdriver Log Collector", "repository": {}, "private": true, @@ -27,10 +27,10 @@ "dependencies": { "@alertlogic/al-collector-js": "3.0.11", "@alertlogic/paws-collector": "2.2.3", - "@google-cloud/logging": "^11.0.0", - "google-proto-files": "^4.2.0", "async": "^3.2.5", "debug": "^4.3.5", + "google-auth-library": "^9.11.0", + "googleapis": "^126.0.0", "moment": "2.30.1" }, "author": "Alert Logic Inc." diff --git a/collectors/googlestackdriver/test/mock.js b/collectors/googlestackdriver/test/mock.js index a4e3d3ae..2b56a6c8 100644 --- a/collectors/googlestackdriver/test/mock.js +++ b/collectors/googlestackdriver/test/mock.js @@ -26,14 +26,27 @@ const AIMS_TEST_CREDS = { secret_key: 'test-secret-key' }; -const LOG_EVENT_JSON_PAYLOAD = { labels: {}, +const LOG_EVENT_JSON_PAYLOAD = { + "insertId": "1qpjficfj5i1ow", + "jsonPayload": { "anotherKey": "RCS test value", "key": "RCS" }, + "resource": { + "type": "global", + "labels": { "project_id": "rcs-test-project-422212" } + }, + "timestamp": "2024-06-19T11:35:55.669107201Z", + "logName": "projects/rcs-test-project-422212/logs/my-json-log", + "receiveTimestamp": "2024-06-19T11:35:55.669107201Z" +}; + +const LOG_EVENT_TEXT_PAYLOAD = { + labels: {}, insertId: '-qwnqhydhp96', httpRequest: null, resource: { labels: [Object], type: 'project' }, timestamp: { seconds: '1577807973', nanos: 776000000 }, severity: 'NOTICE', logName: - 'projects/joe-test-8675309/logs/cloudaudit.googleapis.com%2Factivity', + 'projects/joe-test-8675309/logs/cloudaudit.googleapis.com%2Factivity', operation: null, trace: '', sourceLocation: null, @@ -41,136 +54,282 @@ const LOG_EVENT_JSON_PAYLOAD = { labels: {}, metadata: null, spanId: '', traceSampled: false, - jsonPayload:{ - "fields": { - "resource": { - "structValue": { - "fields": { - "id": { - "stringValue": "7214092729133430404", - "kind": "stringValue" - }, - "region": { - "stringValue": "asia-northeast3", - "kind": "stringValue" - }, - "name": { - "stringValue": "default", - "kind": "stringValue" - }, - "type": { - "stringValue": "subnetwork", - "kind": "stringValue" - } - } - }, - "kind": "structValue" - }, - "event_type": { - "stringValue": "GCE_OPERATION_DONE", - "kind": "stringValue" + textPayload: "An arbitrary payload string", + payload: 'textPayload' +}; + +const LOG_EVENT_PROTO_PAYLOAD = { + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "imran@imranalisyed.com", + "principalSubject": "user:imran@imranalisyed.com" + }, + "requestMetadata": { + "callerIp": "117.247.121.80", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36,gzip(gfe)", + "requestAttributes": { + "time": "2024-04-02T08:17:37.461922381Z", + "auth": {} }, - "trace_id": { - "stringValue": "operation-1579031101569-59c1ed3e5f8fe-e5a0bde3-b85c1f44", - "kind": "stringValue" + "destinationAttributes": {} + }, + "serviceName": "iam.googleapis.com", + "methodName": "google.iam.admin.v1.DisableServiceAccount", + "authorizationInfo": [ + { + "resource": "projects/-/serviceAccounts/109840067714345269337", + "permission": "iam.serviceAccounts.disable", + "granted": true, + "resourceAttributes": { + "name": "projects/-/serviceAccounts/109840067714345269337" + }, + "permissionType": "ADMIN_WRITE" + } + ], + "resourceName": "projects/-/serviceAccounts/109840067714345269337", + "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.DisableServiceAccountRequest", + "name": "projects/imran-49253/serviceAccounts/109840067714345269337" + }, + "response": { "@type": "type.googleapis.com/google.protobuf.Empty" } + }, + "insertId": "5010heeipdi5", + "resource": { + "type": "service_account", + "labels": { + "project_id": "imran-49253", + "unique_id": "109840067714345269337", + "email_id": "im-service-account-private@imran-49253.iam.gserviceaccount.com" + } + }, + "timestamp": "2024-04-02T08:17:37.433885372Z", + "severity": "NOTICE", + "logName": "projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-02T08:17:37.896768264Z" +}; + +const LOG_EVENT_PROTO_PAYLOAD2 = +{ + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "imran@imranalisyed.com", + "principalSubject": "user:imran@imranalisyed.com" + }, + "requestMetadata": { + "callerIp": "117.247.121.80", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36,gzip(gfe)", + "requestAttributes": { + "time": "2024-04-02T08:17:37.461922381Z", + "auth": {} }, - "operation": { - "structValue": { - "fields": { - "name": { - "stringValue": "operation-1579031101569-59c1ed3e5f8fe-e5a0bde3-b85c1f44", - "kind": "stringValue" - }, - "type": { - "stringValue": "operation", - "kind": "stringValue" - }, - "id": { - "stringValue": "192307605219288274", - "kind": "stringValue" + "destinationAttributes": {} + }, + "serviceName": "iam.googleapis.com", + "methodName": "google.iam.admin.v1.DisableServiceAccount", + "authorizationInfo": [ + { + "resource": "projects/-/serviceAccounts/109840067714345269337", + "permission": "iam.serviceAccounts.disable", + "granted": true, + "resourceAttributes": { + "name": "projects/-/serviceAccounts/109840067714345269337" + }, + "permissionType": "ADMIN_WRITE" + } + ], + "resourceName": "projects/-/serviceAccounts/109840067714345269337", + "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.DisableServiceAccountRequest", + "name": "projects/imran-49253/serviceAccounts/109840067714345269337" + }, + "response": { "@type": "type.googleapis.com/google.protobuf.Empty" } + }, + "insertId": "5010heeipdi5", + "resource": { + "type": "service_account", + "labels": { + "project_id": "imran-49253", + "unique_id": "109840067714345269337", + "email_id": "im-service-account-private@imran-49253.iam.gserviceaccount.com" + } + }, + "timestamp": "2024-04-02T08:17:37.433885372Z", + "severity": "NOTICE", + "logName": "projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-02T08:17:37.896768264Z" +}; + +const LOG_EVENT_PROTO_PAYLOAD_FULL_RESPONSE = { + "config": { + "url": "https://logging.googleapis.com/v2/entries:list?filter=timestamp%20%3E%3D%20%222024-04-02T00%3A00%3A00Z%22%0Atimestamp%20%3C%20%222024-05-02T09%3A00%3A00Z%22&pageSize=2&resourceNames=projects%2Fimran-49253", + "method": "POST", + "userAgentDirectives": [ + { + "product": "google-api-nodejs-client", + "version": "7.2.0", + "comment": "gzip" + } + ], + "headers": { + "x-goog-api-client": "gdcl/7.2.0 gl-node/18.20.2", + "Accept-Encoding": "gzip", + "User-Agent": "google-api-nodejs-client/7.2.0 (gzip)", + "Authorization": "Bearer ya29.c.c0AY_VpZhO_4yAOowLfMvaiTiu4_1BUMvHiNgCG18GiH0SZOjd5ZhLQRoB5U4ouyR98R0KjcO5vP8jQ1RZqRONwSUVPmrioCIdOqR1kQsgvm54q7atKqGVlly1xTDQEQpVCfY36mPmzHWfsW2miO_gwNx4E85zyrI7ntIfZ579OV0hxnbl1yJhDooW8oYtl0CWAPXCQ4qZDe7lQdI_ss71al_E-IyA2oJ5fEWFPk72q-hZ03FnBubXyjtlEtSuv7V_FS3Ziy9s9xS9tZjGUC2FR6XMOYLf5N-aPbuawHyk383jnqCT_6wBEqCFS8tFycb5H85NvBoyE7hHHmO4TJMriAKueUbsxXzj48RRDQh1oNMtDLNNkikmsnrn8zFZxVxjqm8XH397A6qUnxbouaU56MY4XafU6B_75hI3JqXrbeu9bk_nYRozbWiwFzFnfM56Fez_YW98ZoOtzv9f7iQMI9r0zt5n3lRtdeqO6kq-6YmtWFWsoYM-umQa7yhlF52VwykwZvfBW-Fwtcm_RY8FnVtdr8c22qUV4gqFlWoSlJSdarIoaS383j8va5m3h_Oyt_iIXYo7l6cy3gRwioh_cFUt0W0llRoYgI--MxSkrsvYcrcg2fXUvb7j8_x2yfelmtUgobwtqV4mqcW0vrSl7Of18t0R2OoR9wJmhxs10YQ7Z2ozsj3Oey5Y1lZYUR5R3BpV3WIXV8rM-m2UbcwU6x2YmyByIsY08cqbOWefqBhZFRksVnlRyiJm-ufsf1wVrJJkfuB6iSWSYmZaq5YngYMmVb7yWMgsoyO64VQi2dR1Vtrc4aM6j2eRefvcwuX1f04OjcpcdogyOkrjkbp6t6UtucapjRUo4h83XtdJdstZW6WcMlOW9BbFvyoi8-WQd33S1UXo2_JphfxFYno-_timh5BsBX-09ZmpaZgSzv19zFx7MS7qMp0o56mJFFXI-bibSOW1deyjXB4_Qv-iOdcn038qjxB8fvejxVSYpScda8Je9OsQi9J" + }, + "params": { + "filter": "timestamp >= \"2024-04-02T00:00:00Z\"\ntimestamp < \"2024-05-02T09:00:00Z\"", + "pageSize": 2, + "resourceNames": ["projects/imran-49253"] + }, + "retry": true, + "responseType": "unknown" + }, + "data": { + "entries": [ + { + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "imran@imranalisyed.com", + "principalSubject": "user:imran@imranalisyed.com" + }, + "requestMetadata": { + "callerIp": "117.247.121.80", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36,gzip(gfe)", + "requestAttributes": { + "time": "2024-04-02T08:17:37.461922381Z", + "auth": {} }, - "region": { - "stringValue": "asia-northeast3", - "kind": "stringValue" + "destinationAttributes": {} + }, + "serviceName": "iam.googleapis.com", + "methodName": "google.iam.admin.v1.DisableServiceAccount", + "authorizationInfo": [ + { + "resource": "projects/-/serviceAccounts/109840067714345269337", + "permission": "iam.serviceAccounts.disable", + "granted": true, + "resourceAttributes": { + "name": "projects/-/serviceAccounts/109840067714345269337" + }, + "permissionType": "ADMIN_WRITE" } + ], + "resourceName": "projects/-/serviceAccounts/109840067714345269337", + "request": { + "@type": "type.googleapis.com/google.iam.admin.v1.DisableServiceAccountRequest", + "name": "projects/imran-49253/serviceAccounts/109840067714345269337" + }, + "response": { "@type": "type.googleapis.com/google.protobuf.Empty" } + }, + "insertId": "5010heeipdi5", + "resource": { + "type": "service_account", + "labels": { + "project_id": "imran-49253", + "unique_id": "109840067714345269337", + "email_id": "im-service-account-private@imran-49253.iam.gserviceaccount.com" } }, - "kind": "structValue" + "timestamp": "2024-04-02T08:17:37.433885372Z", + "severity": "NOTICE", + "logName": "projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-02T08:17:37.896768264Z" }, - "event_subtype": { - "stringValue": "compute.subnetworks.createOrUpdateVirtualSubnetwork", - "kind": "stringValue" - }, - "version": { - "stringValue": "1.2", - "kind": "stringValue" - }, - "event_timestamp_us": { - "stringValue": "1579031109043790", - "kind": "stringValue" - }, - "actor": { - "structValue": { - "fields": { - "user": { - "stringValue": "", - "kind": "stringValue" + { + "protoPayload": { + "@type": "type.googleapis.com/google.cloud.audit.AuditLog", + "status": {}, + "authenticationInfo": { + "principalEmail": "imran@imranalisyed.com", + "principalSubject": "user:imran@imranalisyed.com" + }, + "requestMetadata": { + "callerIp": "117.247.121.80", + "callerSuppliedUserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36,gzip(gfe)", + "requestAttributes": { + "time": "2024-04-02T08:17:44.102276215Z", + "auth": {} + }, + "destinationAttributes": {} + }, + "serviceName": "iam.googleapis.com", + "methodName": "google.iam.admin.v1.EnableServiceAccount", + "authorizationInfo": [ + { + "resource": "projects/-/serviceAccounts/109840067714345269337", + "permission": "iam.serviceAccounts.enable", + "granted": true, + "resourceAttributes": { + "name": "projects/-/serviceAccounts/109840067714345269337" + }, + "permissionType": "ADMIN_WRITE" } + ], + "resourceName": "projects/-/serviceAccounts/109840067714345269337", + "request": { + "name": "projects/imran-49253/serviceAccounts/109840067714345269337", + "@type": "type.googleapis.com/google.iam.admin.v1.EnableServiceAccountRequest" + }, + "response": { "@type": "type.googleapis.com/google.protobuf.Empty" } + }, + "insertId": "tp7fahemm6er", + "resource": { + "type": "service_account", + "labels": { + "project_id": "imran-49253", + "email_id": "im-service-account-private@imran-49253.iam.gserviceaccount.com", + "unique_id": "109840067714345269337" } }, - "kind": "structValue" + "timestamp": "2024-04-02T08:17:44.079997875Z", + "severity": "NOTICE", + "logName": "projects/imran-49253/logs/cloudaudit.googleapis.com%2Factivity", + "receiveTimestamp": "2024-04-02T08:17:46.016142953Z" } - } + ], + "nextPageToken": "EAA4pc_Fx5eu94u6AUrqBSIeIg8KDXRwN2ZhaGVtbTZlcgAqCwio_q6wBhCz15ImSscFCqEFSp4FANAvIAZXvD3Q4F75FqIg-h8Up15-eWH2NS_4P0K-lrH6j5NWGF0U8OXHubt98xgvkXuO1IzppzwkNuoGl7mW-NhhjI-IOgBV9bPbWlD-bnpz7pgpRBWFfILJDfRwlMzxDrV6q6gY-Zg8nNSM2Ae_ADOVtwEUxS_q1BevEuxHQDOEdwA0ke5qMO5xgofmIbFmcIVEmevNOZ-6cJx1SJLPEpoPQqV-hF_jpvCxkXbNUYX93KqHWDx8IbeU9TyJyPeVYciTuK2ptcYe3gJLTRMe7jdiv4EQ_eqgoljTBlhKLUtPT6F6k0ssrQ6KRSltex2bNeQT_fO33svWHVU5AT2q1TkdkbZ9SroeeYWXOnwJqNHNJR4uFcF_Z6393fAqaMdmzX-9rXYNsrsSShhyaSFAlZtWZZ15UBE0cwYaFTVkgzXpe0GuDtuTgBG01SZQRuvntlSxJbASAZIcrBawtyiFfumsw1uUWDEPQiRMk4613a_WNtzbSWK74lOvmdsqWKYaxjUrDmUsZwtQupUsCBqUXBpd83neOGwantHzCr_4CkTqz2azhP-myAXnr90TS_ecb7ho3aDpk7kMI-lOxKsqYUGrJgn4A-kmcjqNlh12vNSHgsknaTKRjevnpGCerP5lt5n9ZakO-BZqAlcFIOHE0nhgKJ9v9yo2yWMv2JmsctzDlmreLkYgJIlBpCqgHkHP7Fwz6j5A8Wb5dbFqndd-gLVGiOR2co4f_BnCPMa1nlLH7VuAXgGO1ZQ4ViSF1e4cfanpvIrJ1WchfDzpF-NRebDQXysWe3wJKImx45VskYPQbxs0LIvBxglVt8eqb5duDyuBVpsWWAVpypWaS-Qa4DasY203NrbPhrJlkRsFad9QVGE0D1bsmNkQD7BS_hIWGgYIgPausAYiDAj_mLSwBhD_k-vcA1DY16Hg2fS6hLABUgcI16S-uJ4eYKKsh4edoIXVfmocCgwI8eOgswYQkrimqgMSCAgBEOLJuNkCGAIgAA" }, - payload: 'jsonPayload' }; - -const LOG_EVENT_TEXT_PAYLOAD = { labels: {}, - insertId: '-qwnqhydhp96', - httpRequest: null, - resource: { labels: [Object], type: 'project' }, - timestamp: { seconds: '1577807973', nanos: 776000000 }, - severity: 'NOTICE', - logName: - 'projects/joe-test-8675309/logs/cloudaudit.googleapis.com%2Factivity', - operation: null, - trace: '', - sourceLocation: null, - receiveTimestamp: { seconds: '1577807974', nanos: 105817306 }, - metadata: null, - spanId: '', - traceSampled: false, - textPayload:"An arbitrary payload string", - payload: 'textPayload' }; - -const LOG_EVENT_PROTO_PAYLOAD = { labels: {}, - insertId: '-qwnqhydhp96', - httpRequest: null, - resource: { labels: [Object], type: 'project' }, - timestamp: { seconds: '1577807973', nanos: 776000000 }, - severity: 'NOTICE', - logName: - 'projects/joe-test-8675309/logs/cloudaudit.googleapis.com%2Factivity', - operation: null, - trace: '', - sourceLocation: null, - receiveTimestamp: { seconds: '1577807974', nanos: 105817306 }, - metadata: null, - spanId: '', - traceSampled: false, - protoPayload: - { type_url: 'type.googleapis.com/google.cloud.audit.AuditLog', - value:{"type":"Buffer","data":[18,0,26,168,2,10,54,117,115,97,99,115,45,115,105,101,109,64,119,101,98,45,99,111,110,115,111,108,101,45,109,105,100,101,111,105,100,46,105,97,109,46,103,115,101,114,118,105,99,101,97,99,99,111,117,110,116,46,99,111,109,42,166,1,47,47,105,97,109,46,103,111,111,103,108,101,97,112,105,115,46,99,111,109,47,112,114,111,106,101,99,116,115,47,119,101,98,45,99,111,110,115,111,108,101,45,109,105,100,101,111,105,100,47,115,101,114,118,105,99,101,65,99,99,111,117,110,116,115,47,117,115,97,99,115,45,115,105,101,109,64,119,101,98,45,99,111,110,115,111,108,101,45,109,105,100,101,111,105,100,46,105,97,109,46,103,115,101,114,118,105,99,101,97,99,99,111,117,110,116,46,99,111,109,47,107,101,121,115,47,48,48,53,51,56,57,57,98,50,49,56,49,54,97,102,52,99,53,100,100,101,50,100,54,102,54,50,56,56,49,52,100,100,48,101,97,53,98,51,98,66,69,115,101,114,118,105,99,101,65,99,99,111,117,110,116,58,117,115,97,99,115,45,115,105,101,109,64,119,101,98,45,99,111,110,115,111,108,101,45,109,105,100,101,111,105,100,46,105,97,109,46,103,115,101,114,118,105,99,101,97,99,99,111,117,110,116,46,99,111,109,34,65,10,12,51,46,50,51,55,46,49,55,46,49,54,51,18,29,103,114,112,99,45,110,111,100,101,45,106,115,47,49,46,49,48,46,48,44,103,122,105,112,40,103,102,101,41,58,16,74,12,8,192,159,179,176,6,16,148,129,149,153,2,106,0,66,0,58,22,108,111,103,103,105,110,103,46,103,111,111,103,108,101,97,112,105,115,46,99,111,109,66,49,103,111,111,103,108,101,46,108,111,103,103,105,110,103,46,118,50,46,76,111,103,103,105,110,103,83,101,114,118,105,99,101,86,50,46,76,105,115,116,76,111,103,69,110,116,114,105,101,115,74,109,10,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,18,23,108,111,103,103,105,110,103,46,108,111,103,69,110,116,114,105,101,115,46,108,105,115,116,24,1,42,52,10,22,108,111,103,103,105,110,103,46,103,111,111,103,108,101,97,112,105,115,46,99,111,109,18,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,74,116,10,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,18,30,108,111,103,103,105,110,103,46,112,114,105,118,97,116,101,76,111,103,69,110,116,114,105,101,115,46,108,105,115,116,24,1,42,52,10,22,108,111,103,103,105,110,103,46,103,111,111,103,108,101,97,112,105,115,46,99,111,109,18,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,90,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,130,1,238,1,10,21,10,8,112,97,103,101,83,105,122,101,18,9,17,0,0,0,0,0,64,143,64,10,49,10,13,114,101,115,111,117,114,99,101,78,97,109,101,115,18,32,50,30,10,28,26,26,111,114,103,97,110,105,122,97,116,105,111,110,115,47,54,50,50,56,53,50,55,51,55,50,57,52,10,90,10,6,102,105,108,116,101,114,18,80,26,78,116,105,109,101,115,116,97,109,112,32,62,61,32,34,50,48,50,52,45,48,52,45,48,51,84,48,51,58,50,51,58,48,54,46,48,48,48,90,34,10,116,105,109,101,115,116,97,109,112,32,60,32,34,50,48,50,52,45,48,52,45,48,51,84,48,51,58,50,52,58,48,54,46,48,48,48,90,34,10,70,10,5,64,116,121,112,101,18,61,26,59,116,121,112,101,46,103,111,111,103,108,101,97,112,105,115,46,99,111,109,47,103,111,111,103,108,101,46,108,111,103,103,105,110,103,46,118,50,46,76,105,115,116,76,111,103,69,110,116,114,105,101,115,82,101,113,117,101,115,116]}, - }, - payload: 'protoPayload' }; + "headers": { + "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", + "cache-control": "private", + "connection": "close", + "content-encoding": "gzip", + "content-type": "application/json; charset=UTF-8", + "date": "Tue, 11 Jun 2024 11:16:03 GMT", + "server": "ESF", + "server-timing": "gfet4t7; dur=1760", + "transfer-encoding": "chunked", + "vary": "Origin, X-Origin, Referer", + "x-content-type-options": "nosniff", + "x-frame-options": "SAMEORIGIN", + "x-xss-protection": "0" + }, + "status": 200, + "statusText": "OK", + "request": { + "responseURL": "https://logging.googleapis.com/v2/entries:list?filter=timestamp%20%3E%3D%20%222024-04-02T00%3A00%3A00Z%22%0Atimestamp%20%3C%20%222024-05-02T09%3A00%3A00Z%22&pageSize=2&resourceNames=projects%2Fimran-49253" + } +}; const FUNCTION_ARN = 'arn:aws:lambda:us-east-1:352283894008:function:test-01-CollectLambdaFunction-2CWNLPPW5XO8'; const FUNCTION_NAME = 'test-TestCollectLambdaFunction-1JNNKQIPOTEST'; - +const MOCK_ENTRIES = { + list: () => { } +}; module.exports = { AIMS_TEST_CREDS: AIMS_TEST_CREDS, FUNCTION_ARN: FUNCTION_ARN, FUNCTION_NAME: FUNCTION_NAME, LOG_EVENT_JSON_PAYLOAD, LOG_EVENT_TEXT_PAYLOAD, - LOG_EVENT_PROTO_PAYLOAD + LOG_EVENT_PROTO_PAYLOAD, + LOG_EVENT_PROTO_PAYLOAD2, + LOG_EVENT_PROTO_PAYLOAD_FULL_RESPONSE, + MOCK_ENTRIES: MOCK_ENTRIES }; diff --git a/collectors/googlestackdriver/test/test.js b/collectors/googlestackdriver/test/test.js index 29ef0fc0..465681a8 100644 --- a/collectors/googlestackdriver/test/test.js +++ b/collectors/googlestackdriver/test/test.js @@ -1,42 +1,60 @@ const assert = require('assert'); const sinon = require('sinon'); const moment = require('moment'); -const logging = require('@google-cloud/logging'); +const { google } = require('googleapis'); const { CloudWatch } = require("@aws-sdk/client-cloudwatch"), { KMS } = require("@aws-sdk/client-kms"), { SSM } = require("@aws-sdk/client-ssm"); const googlestackdriverMock = require('./mock'); +const logEntriesService = google.logging('v2'); var GooglestackdriverCollector = require('../collector').GooglestackdriverCollector; +const { auth } = require("google-auth-library"); +let mockauthenticationObject; +let mocklogEntriesServiceObject; + +function setAlServiceStub() { + mockauthenticationObject = sinon.stub(auth, 'fromJSON').callsFake( + function fakeFn(path) { + return {}; + }); +} +function restoreAlServiceStub() { + mockauthenticationObject.restore(); +} var logginClientStub = {}; function setLoggingClientStub(){ - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); + logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { - res([ - [ - googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD, - googlestackdriverMock.LOG_EVENT_TEXT_PAYLOAD, - googlestackdriverMock.LOG_EVENT_JSON_PAYLOAD - ], - 'http://somenextpage.com' - ]); + res({ + data: { + entries: [ + googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD2, + googlestackdriverMock.LOG_EVENT_TEXT_PAYLOAD, + googlestackdriverMock.LOG_EVENT_JSON_PAYLOAD + ], + nextPageToken: 'http://somenextpage.com' + } + }); }); }); logginClientStub.onCall(1).callsFake(() => { return new Promise((res, rej) => { - res([ - [ - googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD, - googlestackdriverMock.LOG_EVENT_TEXT_PAYLOAD, - googlestackdriverMock.LOG_EVENT_JSON_PAYLOAD - ], - null - ]); + res({ + data: { + entries: [ + googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD, + googlestackdriverMock.LOG_EVENT_TEXT_PAYLOAD, + googlestackdriverMock.LOG_EVENT_JSON_PAYLOAD + ], + nextPageToken: null + } + }); }); }); } @@ -58,9 +76,17 @@ describe('Unit Tests', function() { }; return callback(null, data); }); + logEntriesService.entries = googlestackdriverMock.MOCK_ENTRIES; + mocklogEntriesServiceObject = sinon.stub(google, 'logging').callsFake( + function fakeFn(path) { + return logEntriesService; + }); + setAlServiceStub(); }); afterEach(function(){ + restoreAlServiceStub(); + mocklogEntriesServiceObject.restore(); KMS.prototype.decrypt.restore(); SSM.prototype.getParameter.restore(); }); @@ -138,8 +164,7 @@ describe('Unit Tests', function() { }); it('Get Logs Cloudy', function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { rej("Here is an error"); @@ -164,8 +189,7 @@ describe('Unit Tests', function() { }); it('Get Logs check API Throttling', function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { rej({code:8, @@ -197,9 +221,8 @@ describe('Unit Tests', function() { }); it(`Get Logs check API Throttling when going through pagination then check if it reduce page size and able to fetch the data`, function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - - logginClientStub.onCall(0).callsFake(() => { + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); + logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { rej({code: 8, details: 'Received message larger than max (4776477 vs. 4194304)'}); @@ -220,7 +243,7 @@ describe('Unit Tests', function() { var reportSpy = sinon.spy(collector, 'reportApiThrottling'); let putMetricDataStub = sinon.stub(CloudWatch.prototype, 'putMetricData').callsFake((params, callback) => callback()); collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) =>{ - assert.equal(moment(newState.until).diff(newState.since, 'seconds'), 15); + assert.equal(moment(newState.until).diff(newState.since, 'seconds'), 15); assert.equal(true, reportSpy.calledOnce); assert.equal(newState.nextPage.pageSize, 500); assert.equal(logs.length, 0); @@ -231,14 +254,13 @@ describe('Unit Tests', function() { }); }); }); - + it(`Get Logs check API Throttling with 'Received message larger than max (4776477 vs. 4194304)' for time interval less than 15 sec then check with reduce page size able to fetch the data`, function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { rej({code: 8, - details: 'Received message larger than max (4776477 vs. 4194304)'}); + message: 'Received message larger than max (4776477 vs. 4194304)'}); }); }); @@ -267,29 +289,33 @@ describe('Unit Tests', function() { }); it('Stops paginiating at the pagination limit', function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); - - const nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"] }; + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); + const startDate = moment().subtract(3, 'days'); + let since = startDate.toISOString(); + let until = startDate.add(2, 'days').toISOString(); + const filter = `timestamp >= "${since}" +timestamp < "${until}"`; + let nextPage = { pageToken: 'http://somenextpage.com', "pageSize": 1000, "resourceNames": ["projects/a-fake-project"], filter }; + logginClientStub.callsFake(() => { return new Promise((res, rej) => { - res([ - [ - googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD - ], - nextPage - ]); + res({ + data: { + entries: [googlestackdriverMock.LOG_EVENT_PROTO_PAYLOAD2], + nextPageToken: 'http://somenextpage.com' + } + }); }); }); GooglestackdriverCollector.load().then(function(creds) { var collector = new GooglestackdriverCollector(ctx, creds); - const startDate = moment().subtract(3, 'days'); const curState = { stream: "projects/a-fake-project", - since: startDate.toISOString(), - until: startDate.add(2, 'days').toISOString(), + since, + until, poll_interval_sec: 1 }; - + collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) =>{ assert.ok(logginClientStub.calledTwice); assert.equal(logs.length, parseInt(process.env.paws_max_pages_per_invocation)); @@ -301,7 +327,7 @@ describe('Unit Tests', function() { }); it('Get Logs check client error', function(done) { - logginClientStub = sinon.stub(logging.v2.LoggingServiceV2Client.prototype, 'listLogEntries'); + logginClientStub = sinon.stub(logEntriesService.entries, 'list'); logginClientStub.onCall(0).callsFake(() => { return new Promise((res, rej) => { @@ -449,9 +475,7 @@ describe('Unit Tests', function() { fmt.messageTypeId, googlestackdriverMock. LOG_EVENT_JSON_PAYLOAD. - jsonPayload. - fields. - event_type.stringValue + jsonPayload ); done(); }); @@ -501,11 +525,10 @@ describe('Unit Tests', function() { fmt.messageTypeId, googlestackdriverMock. LOG_EVENT_PROTO_PAYLOAD. - protoPayload. - type_url + protoPayload['@type'] ); done(); }); }); }); -}); + }); \ No newline at end of file diff --git a/collectors/gsuite/test/gsuite_test.js b/collectors/gsuite/test/gsuite_test.js index 7fb76277..7392c656 100644 --- a/collectors/gsuite/test/gsuite_test.js +++ b/collectors/gsuite/test/gsuite_test.js @@ -15,17 +15,17 @@ const { CloudWatch } = require("@aws-sdk/client-cloudwatch"), var responseStub = {}; let listEvent; let listAlert; -let authenticationT; +let mockAuthenticationObject; function setAlServiceStub() { - authenticationT = sinon.stub(auth, 'fromJSON').callsFake( + mockAuthenticationObject = sinon.stub(auth, 'fromJSON').callsFake( function fakeFn(path) { return {}; }); } function restoreAlServiceStub() { - authenticationT.restore(); + mockAuthenticationObject.restore(); } describe('Unit Tests', function () { diff --git a/ps_spec.yml b/ps_spec.yml index e593d779..0b3e557c 100644 --- a/ps_spec.yml +++ b/ps_spec.yml @@ -177,7 +177,7 @@ stages: - ./build_collector.sh googlestackdriver env: ALPS_SERVICE_NAME: "paws-googlestackdriver-collector" - ALPS_SERVICE_VERSION: "1.2.11" #set the value from collector package json + ALPS_SERVICE_VERSION: "1.2.12" #set the value from collector package json outputs: file: ./googlestackdriver-collector* packagers: