diff --git a/collectors/ciscomeraki/.jshintrc b/collectors/ciscomeraki/.jshintrc new file mode 100644 index 00000000..d847dead --- /dev/null +++ b/collectors/ciscomeraki/.jshintrc @@ -0,0 +1,15 @@ +{ + "esversion": 8, + "shadow": "outer", + "undef": true, + "unused": "vars", + "node": true, + "predef": [ + "it", + "describe", + "before", + "after", + "beforeEach", + "afterEach" + ] +} diff --git a/collectors/ciscomeraki/.npmrc b/collectors/ciscomeraki/.npmrc new file mode 100644 index 00000000..43c97e71 --- /dev/null +++ b/collectors/ciscomeraki/.npmrc @@ -0,0 +1 @@ +package-lock=false diff --git a/collectors/ciscomeraki/LICENSE b/collectors/ciscomeraki/LICENSE new file mode 100644 index 00000000..6eac43de --- /dev/null +++ b/collectors/ciscomeraki/LICENSE @@ -0,0 +1,8 @@ +Copyright 2019 ALERT LOGIC + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/collectors/ciscomeraki/Makefile b/collectors/ciscomeraki/Makefile new file mode 100644 index 00000000..2bc6a0c3 --- /dev/null +++ b/collectors/ciscomeraki/Makefile @@ -0,0 +1,6 @@ +AWS_LAMBDA_FUNCTION_NAME ?= alertlogic-ciscomeraki-collector +AWS_LAMBDA_PACKAGE_NAME ?= al-ciscomeraki-collector.zip +AWS_LAMBDA_CONFIG_PATH ?= ./al-ciscomeraki-collector.json +AWS_CFN_TEMPLATE_PATH ?= ./cfn/ciscomeraki-collector.template + +-include ../collector.mk diff --git a/collectors/ciscomeraki/README.md b/collectors/ciscomeraki/README.md new file mode 100644 index 00000000..96d120e7 --- /dev/null +++ b/collectors/ciscomeraki/README.md @@ -0,0 +1,149 @@ +# Ciscomeraki collector +Alert Logic Ciscomeraki AWS Based API Poll (PAWS) Log Collector Library. + +# Overview +This repository contains the AWS JavaScript Lambda function and CloudFormation +Template (CFT) for deploying a log collector in AWS which will poll Ciscomeraki (Network Events) service API to collect and +forward logs to the Alert Logic CloudInsight backend services. + +# Installation + +# Cisco Meraki Dashboard Log Collection Setup + +Instructions for setting up log collection from Cisco Meraki Dashboard using its API. + +## Prerequisites + +1. **Cisco Meraki Dashboard Account**: You need to have access to a Cisco Meraki Dashboard account with administrative privileges. +2. **API Key**: Generate an API key from the Cisco Meraki Dashboard. Follow [this guide](https://developer.cisco.com/meraki/api-v1/authorization/) to obtain your API key. + +## Setup Steps + +1. **Enable API Access**: + - Log in to your [Cisco Meraki Dashboard](https://dashboard.meraki.com) account. + - You need to have access to organizational level administrative privileges. + - Get the Organization ID at the bottom of the page ![ScreenShot](./docs/Ciscomerakiorg.png). + - Navigate to *Organization > Settings*. + - Under *Dashboard API access*, enable API access ![ScreenShot](./docs/Ciscomeraki_img1.png). + +2. **Generate API Key**: + - Go to *My Profile*. + - Under *API access*, generate a new API key ![ScreenShot](./docs/Ciscomeraki_img2.png). + +3. **Collect Network Events**: + - Refer to the [Cisco Meraki Dashboard API documentation](https://developer.cisco.com/meraki/api/) for details on how to use the API to retrieve [network events](https://developer.cisco.com/meraki/api-v1/get-network-events/). + - Example API endpoint: `GET /organizations/{organizationId}/networks/{networkId}/events` + +4. **Handle Authentication**: + - Include your API key in the request headers for authentication. + - Example: `X-Cisco-Meraki-API-Key: YOUR_API_KEY` + +### Rate Limits +- Cisco Meraki Dashboard imposes rate limits on API requests to ensure fair usage and prevent abuse. Refer to the [Rate Limits documentation](https://developer.cisco.com/meraki/api-v1/rate-limit/) for details on the specific limits for different endpoints. +- Ensure that your application adheres to these rate limits to avoid throttling errors. + +### Organization Call Budget +- Each Meraki organization has a call budget of **10 requests per second**, regardless of the number of API applications interacting with that organization. + +### Throttling Errors +- Throttling errors occur when your API requests exceed the allowed rate limit. When a throttling error occurs, the API will return an HTTP 429 status code along with an error message. +- Refer to the [Throttling Errors documentation](https://developer.cisco.com/meraki/api/#/rest/guides/throttling-errors) for information on how to handle throttling errors and retry mechanisms. + +### 2. API Docs + +1. [Network Events](https://developer.cisco.com/meraki/api-v1/get-network-events/) + + This endpoint allows you to retrieve network events from **all networks** within an organization, filtered by specific product types such as "appliance", "switch", and more. These events provide insights into network changes, device status updates, and other relevant activities. + + +API URLs required for Ciscomeraki collector for Example + +| URL | +|--------------------------------------| +| https://n219.meraki.com/organizations/{organizationId}/networks/{networkId}/events | + +## Support + +If you encounter any issues or have questions, please reach out to Cisco Meraki support or refer to their documentation. + +Refer to [CF template readme](./cfn/README.md) for installation instructions. + +# How it works + +## Update Trigger + +The `Updater` is a timer triggered function that runs a deployment sync operation +every 12 hours in order to keep the collector lambda function up to date. +The `Updater` syncs from the Alert Logic S3 bucket where you originally deployed from. + +## Collection Trigger + +The `Collector` function is an AWS lambda function which is triggered by SQS which contains collection state message. +During each invocation the function polls 3rd party service log API and sends retrieved data to +AlertLogic `Ingest` service for further processing. + +## Checkin Trigger + +The `Checkin` Scheduled Event trigger is used to report the health and status of +the Alert Logic AWS lambda collector to the `Azcollect` back-end service based on +an AWS Scheduled Event that occurs every 15 minutes. + + +# Development + +## Creating New Collector Types +run `npm run create-collector <> <>` to create a skeleton collector in the `collectors` folder. + +## Build collector +Clone this repository and build a lambda package by executing: +``` +$ git clone https://github.com/alertlogic/paws-collector.git +$ cd paws-collector/collectors/ciscomeraki +$ make deps test package +``` + +The package name is *al-ciscomeraki-collector.zip* + +## Debugging + +To get a debug trace, set an Node.js environment variable called DEBUG and +specify the JavaScript module/s to debug. + +E.g. + +``` +export DEBUG=* +export DEBUG=index +``` + +Or set an environment variable called "DEBUG" in your AWS stack (using the AWS +console) for a collector AWS Lambda function, with value "index" or "\*". + +See [debug](https://www.npmjs.com/package/debug) for further details. + +## Invoking locally + +In order to invoke lambda locally please follow the [instructions](https://docs.aws.amazon.com/lambda/latest/dg/sam-cli-requirements.html) to install AWS SAM. +AWS SAM uses `default` credentials profile from `~/.aws/credentials`. + + 1. Encrypt the key using aws cli: +``` +aws kms encrypt --key-id KMS_KEY_ID --plaintext AIMS_SECRET_KEY +``` + 2. Include the encrypted token, and `KmsKeyArn` that you used in Step 1 inside my SAM yaml: +``` + KmsKeyArn: arn:aws:kms:us-east-1:xxx:key/yyy + Environment: + Variables: +``` + 3. Fill in environment variables in `env.json` (including encrypted AIMS secret key) and invoke locally: + +``` +cp ./local/env.json.tmpl ./local/env.json +vi ./local/env.json +make test +make sam-local +``` + 4. Please see `local/event.json` for the event payload used for local invocation. +Please write your readme here + diff --git a/collectors/ciscomeraki/al-ciscomeraki-collector.json b/collectors/ciscomeraki/al-ciscomeraki-collector.json new file mode 100644 index 00000000..650b2fcb --- /dev/null +++ b/collectors/ciscomeraki/al-ciscomeraki-collector.json @@ -0,0 +1,6 @@ +{ + "Runtime": { + "path": "Runtime", + "value": "nodejs18.x" + } +} diff --git a/collectors/ciscomeraki/cfn/ciscomeraki-collector.template b/collectors/ciscomeraki/cfn/ciscomeraki-collector.template new file mode 100644 index 00000000..2ea14cd1 --- /dev/null +++ b/collectors/ciscomeraki/cfn/ciscomeraki-collector.template @@ -0,0 +1,115 @@ +{ + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Alert Logic template for creating a CiscoMeraki log collector", + "Parameters": { + "AlertlogicAccessKeyId": { + "Description": "Alert Logic Access Key Id obtained from AIMS", + "Type": "String" + }, + "AlertlogicSecretKey": { + "Description": "Alert Logic Secret Key returned from AIMS for the Access Key Id", + "Type": "String", + "NoEcho": true + }, + "AlApplicationId": { + "Description": "Alert Logic Application Id for collector logs", + "Type": "String", + "Default": "ciscomeraki" + }, + "AlApiEndpoint": { + "Description": "Alert Logic API endpoint", + "Type": "String", + "Default": "api.global-services.global.alertlogic.com", + "AllowedValues": [ + "api.global-services.global.alertlogic.com", + "api.global-integration.product.dev.alertlogic.com" + ] + }, + "AlDataResidency": { + "Description": "Alert Logic Data Residency", + "Type": "String", + "Default": "default", + "AllowedValues": ["default"] + }, + "PackagesBucketPrefix": { + "Description": "S3 bucket name prefix where collector packages are located.", + "Type": "String", + "Default": "alertlogic-collectors" + }, + "PawsCollectorTypeName": { + "Description": "A collector type name. For example, okta, auth0", + "Type": "String", + "Default": "ciscomeraki" + }, + "AlertlogicCustomerId": { + "Description": "Optional. Alert Logic customer ID which collected data should be reported for. If not set customer ID is derived from AIMs tokens", + "Type": "String", + "Default": "" + }, + "CollectorId": { + "Description": "Optional. A collector UUID if known.", + "Type": "String", + "Default": "none" + }, + "CiscoMerakiEndpoint": { + "Description": "Cisco Meraki API URL to poll. For Example https://n149.meraki.com/api/v1", + "Type": "String" + }, + "CiscoMerakiClientId": { + "Description": "Cisco Meraki Client ID for oauth2 authentication type", + "Type": "String" + }, + "CiscoMerakiSecret": { + "Description": "Cisco Meraki Client Secret API Key for authentication.", + "Type": "String", + "NoEcho": true + }, + "CiscoMerakiOrgKey": { + "Description": "Cisco Meraki Organisation Key, used for collection of network events", + "Type": "String", + "Default": "" + }, + "CiscoMerakiProductTypes": { + "Description": "Define Product Types. Please pass JSON formatted list of product types. Possible values are [\"appliance\",\"switch\",\"systemsManager\"]", + "Type": "String", + "Default": "" + }, + "CollectionStartTs": { + "Description": "Timestamp when log collection starts. For example, 2019-11-21T16:00:00Z", + "Type": "String", + "Default" : "", + "AllowedPattern" : "(?:^\\d{4}(-\\d{2}){2}T(\\d{2}:){2}\\d{2}Z$)?" + } + }, + "Resources":{ + "CiscoMerakiCollectorStack" : { + "Type" : "AWS::CloudFormation::Stack", + "Properties" : { + "TemplateURL" : {"Fn::Join" : ["", [ + "https://s3.amazonaws.com/", + {"Ref":"PackagesBucketPrefix"}, "-us-east-1", + "/cfn/paws-collector.template" + ]]}, + "Parameters" : { + "AlertlogicAccessKeyId" : { "Ref":"AlertlogicAccessKeyId" }, + "AlertlogicSecretKey" : { "Ref":"AlertlogicSecretKey" }, + "AlApplicationId" : { "Ref":"AlApplicationId" }, + "AlApiEndpoint" : { "Ref":"AlApiEndpoint" }, + "AlDataResidency" : { "Ref":"AlDataResidency" }, + "PackagesBucketPrefix" : { "Ref":"PackagesBucketPrefix" }, + "PawsCollectorTypeName" : "ciscomeraki", + "AlertlogicCustomerId" : { "Ref":"AlertlogicCustomerId" }, + "CollectorId" : { "Ref":"CollectorId" }, + "PollingInterval" : 60, + "PawsEndpoint" : { "Ref":"CiscoMerakiEndpoint" }, + "PawsAuthType" : "oauth2", + "PawsClientId" : "", + "CollectorParamString1" : { "Ref":"CiscoMerakiProductTypes" }, + "CollectorParamString2" : { "Ref":"CiscoMerakiOrgKey" }, + "PawsSecret" : { "Ref":"CiscoMerakiSecret" }, + "CollectionStartTs" : { "Ref":"CollectionStartTs" } + } + } + } + } +} diff --git a/collectors/ciscomeraki/collector.js b/collectors/ciscomeraki/collector.js new file mode 100644 index 00000000..cc5a4c1e --- /dev/null +++ b/collectors/ciscomeraki/collector.js @@ -0,0 +1,248 @@ +/* ----------------------------------------------------------------------------- + * @copyright (C) 2024, Alert Logic, Inc + * @doc + * + * ciscomeraki class. + * + * @end + * ----------------------------------------------------------------------------- + */ +'use strict'; + +const moment = require('moment'); +const PawsCollector = require('@alertlogic/paws-collector').PawsCollector; +const parse = require('@alertlogic/al-collector-js').Parse; +const packageJson = require('./package.json'); +const calcNextCollectionInterval = require('@alertlogic/paws-collector').calcNextCollectionInterval; +const utils = require("./utils"); +const AlLogger = require('@alertlogic/al-aws-collector-js').Logger; +const MAX_POLL_INTERVAL = 900; +const API_THROTTLING_ERROR = 429; +const API_NOT_FOUND_ERROR = 404; + +let typeIdPaths = [{ path: ["type"] }]; +let tsPaths = [{ path: ["occurredAt"] }]; + +class CiscomerakiCollector extends PawsCollector { + constructor(context, creds) { + super(context, creds, packageJson.version); + } + + async pawsInitCollectionState(event, callback) { + let collector = this; + const resourceNames = process.env.collector_streams ? JSON.parse(process.env.collector_streams) : []; + try { + if (resourceNames.length > 0) { + const initialStates = this.generateInitialStates(resourceNames); + return callback(null, initialStates, 1); + } + else { + try { + const payloadObj = utils.getOrgKeySecretEndPoint(collector.secret); + let networks = await utils.getAllNetworks(payloadObj); + if (networks.length > 0) { + const initialStates = this.generateInitialStates(networks); + return callback(null, initialStates, 1); + } else { + return callback("Error: No networks found"); + } + } catch (error) { + return callback(error); + } + } + + } catch (error) { + return callback(error); + } + } + + generateInitialStates(networkIds) { + const startTs = process.env.paws_collection_start_ts ? + process.env.paws_collection_start_ts : + moment().toISOString(); + const endTs = moment(startTs).add(this.pollInterval, 'seconds').toISOString(); + const initialStates = networkIds.map(networkId => ({ + networkId: networkId, + since: startTs, + until: endTs, + nextPage: null, + poll_interval_sec: parseInt(Math.floor(Math.random() * 30) + 1) + })); + return initialStates; + } + + async handleEvent(event) { + let collector = this; + let context = this._invokeContext; + let parsedEvent = collector._parseEvent(event); + + switch (parsedEvent.RequestType) { + case 'ScheduledEvent': + switch (parsedEvent.Type) { + case 'SelfUpdate': + await collector.handleUpdateStreamsFromNetworks(); + return collector.handleUpdate(); + break; + default: + super.handleEvent(event); + } + default: + super.handleEvent(event); + } + } + + async handleUpdateStreamsFromNetworks() { + let collector = this; + try { + const payloadObj = utils.getOrgKeySecretEndPoint(collector.secret); + //get networks from api + let networks = await utils.getAllNetworks(payloadObj); + const keyValue = `${process.env.customer_id}/${collector._pawsCollectorType}/${collector.collector_id}/networks_${collector.collector_id}.json`; + let params = await utils.getS3ObjectParams(keyValue, undefined); + + //get networks json from s3 bucket + let networksFromS3 = await utils.fetchJsonFromS3Bucket(params.bucketName, params.key); + AlLogger.debug(`CMRI0000025 networks: ${JSON.stringify(networks)} networksFromS3 ${JSON.stringify(params)} ${JSON.stringify(networksFromS3)}`); + if (networks.length > 0 && Array.isArray(networksFromS3) && networksFromS3.length > 0) { + let differenceNetworks = utils.differenceOfNetworksArray(networks, networksFromS3); + AlLogger.debug(`CMRI0000024 Networks updated ${JSON.stringify(differenceNetworks)}`); + + if (differenceNetworks.length > 0) { + const initialStates = this.generateInitialStates(differenceNetworks); + AlLogger.debug(`CMRI0000020: SQS message added ${JSON.stringify(initialStates)}`); + collector._storeCollectionState({}, initialStates, this.pollInterval, async () => { + await utils.uploadNetworksListInS3Bucket(keyValue, networks); + }); + } + } else if (networksFromS3 && (networksFromS3.Code === 'NoSuchKey' || networksFromS3.Code === 'AccessDenied')) { + AlLogger.debug(`CMRI0000026 networks ${JSON.stringify(params)} ${JSON.stringify(networks)}`); + await utils.uploadNetworksListInS3Bucket(keyValue, networks); + } + } catch (error) { + AlLogger.debug(`Error updating streams from networks: ${error.message}`); + } + } + + pawsGetLogs(state, callback) { + const collector = this; + + const productTypes = process.env.paws_collector_param_string_1 ? JSON.parse(process.env.paws_collector_param_string_1) : []; + if (!productTypes) { + return callback("The Product Types was not found!"); + } + const { clientSecret, apiEndpoint, orgKey } = utils.getOrgKeySecretEndPoint(collector.secret, callback); + const apiDetails = utils.getAPIDetails(orgKey, productTypes); + if (!apiDetails.url) { + return callback("The API name was not found!"); + } + AlLogger.info(`CMRI000001 Collecting data for NetworkId-${state.networkId} from ${state.since}`); + utils.getAPILogs(apiDetails, [], apiEndpoint, state, clientSecret, process.env.paws_max_pages_per_invocation) + .then(({ accumulator, nextPage }) => { + let newState; + if (nextPage === undefined) { + newState = this._getNextCollectionState(state); + } else { + newState = this._getNextCollectionStateWithNextPage(state, nextPage); + } + AlLogger.info(`CMRI000002 Next collection in ${newState.poll_interval_sec} seconds`); + return callback(null, accumulator, newState, newState.poll_interval_sec); + }) + .catch((error) => { + if (error && error.response && error.response.status == API_THROTTLING_ERROR) { + collector.handleThrottlingError(error, state, callback); + } else { + collector.handleOtherErrors(error, state, callback); + } + }); + } + + handleThrottlingError(error, state, callback) { + const maxRandom = 5; + let retry = parseInt(error.response.headers['retry-after']) || 1; + retry += Math.floor(Math.random() * (maxRandom + 1)); + state.poll_interval_sec = state.poll_interval_sec < MAX_POLL_INTERVAL ? + parseInt(state.poll_interval_sec) + retry : MAX_POLL_INTERVAL; + AlLogger.info(`CMRI000007 Throttling error, retrying after ${state.poll_interval_sec} sec`); + this.reportApiThrottling(function () { + return callback(null, [], state, state.poll_interval_sec); + }); + } + + handleOtherErrors(error, state, callback) { + if (error && error.response && error.response.status == API_NOT_FOUND_ERROR) { + state.retry = state.retry ? state.retry + 1 : 1; + if (state.retry > 3) { + AlLogger.debug(`CMRI0000021 Deleted SQS message from Queue${JSON.stringify(state)}`); + this._invokeContext.succeed(); + } else { + return callback(error); + } + } else if (error && error.response && error.response.data) { + AlLogger.debug(`CMRI0000022 error ${error.response.data.errors} - status: ${error.response.status}`); + error.response.data.errorCode = error.response.status; + return callback(error.response.data); + } else { + return callback(error); + } + } + + _getNextCollectionState(curState) { + const untilMoment = moment(curState.until); + const { nextUntilMoment, nextSinceMoment, nextPollInterval } = calcNextCollectionInterval('no-cap', untilMoment, this.pollInterval); + return { + networkId: curState.networkId, + since: nextSinceMoment.toISOString(), + until: nextUntilMoment.toISOString(), + nextPage: null, + poll_interval_sec: nextPollInterval + }; + } + + _getNextCollectionStateWithNextPage({ networkId, since, until }, nextPage) { + const obj = { + networkId, + since: nextPage, + until, + nextPage: null, + poll_interval_sec: 1 + }; + return obj; + } + + pawsGetRegisterParameters(event, callback) { + const regValues = { + ciscoMerakiObjectNames: process.env.collector_streams + }; + callback(null, regValues); + } + + pawsFormatLog(msg) { + // TODO: double check that this message parsing fits your use case + let collector = this; + + let ts = parse.getMsgTs(msg, tsPaths); + let typeId = parse.getMsgTypeId(msg, typeIdPaths); + + let formattedMsg = { + hostname: collector.collector_id, + messageTs: ts.sec, + priority: 11, + progName: 'CiscomerakiCollector', + message: JSON.stringify(msg), + messageType: 'json/ciscomeraki', + application_id: collector.application_id + }; + + if (typeId !== null && typeId !== undefined) { + formattedMsg.messageTypeId = `${typeId}`; + } + if (ts.usec) { + formattedMsg.messageTsUs = ts.usec; + } + return formattedMsg; + } +} + +module.exports = { + CiscomerakiCollector: CiscomerakiCollector +} \ No newline at end of file diff --git a/collectors/ciscomeraki/docs/Ciscomeraki_img1.png b/collectors/ciscomeraki/docs/Ciscomeraki_img1.png new file mode 100644 index 00000000..c7ff2b9f Binary files /dev/null and b/collectors/ciscomeraki/docs/Ciscomeraki_img1.png differ diff --git a/collectors/ciscomeraki/docs/Ciscomeraki_img2.png b/collectors/ciscomeraki/docs/Ciscomeraki_img2.png new file mode 100644 index 00000000..955046fe Binary files /dev/null and b/collectors/ciscomeraki/docs/Ciscomeraki_img2.png differ diff --git a/collectors/ciscomeraki/docs/Ciscomerakiorg.png b/collectors/ciscomeraki/docs/Ciscomerakiorg.png new file mode 100644 index 00000000..998f9098 Binary files /dev/null and b/collectors/ciscomeraki/docs/Ciscomerakiorg.png differ diff --git a/collectors/ciscomeraki/index.js b/collectors/ciscomeraki/index.js new file mode 100644 index 00000000..c36b4f98 --- /dev/null +++ b/collectors/ciscomeraki/index.js @@ -0,0 +1,23 @@ +/* ----------------------------------------------------------------------------- + * @copyright (C) 2019, Alert Logic, Inc + * @doc + * + * Ciscomeraki System logs extension. + * + * @end + * ----------------------------------------------------------------------------- + */ + +const debug = require('debug') ('index'); +const AlLogger = require('@alertlogic/al-aws-collector-js').Logger; +const CiscomerakiCollector = require('./collector').CiscomerakiCollector; + +exports.handler = CiscomerakiCollector.makeHandler(function(event, context) { + debug('input event: ', event); + AlLogger.defaultMeta = { requestId: context.awsRequestId }; + var ciscomerakic; + CiscomerakiCollector.load().then(function(creds) { + ciscomerakic = new CiscomerakiCollector(context, creds); + ciscomerakic.handleEvent(event); + }); +}); diff --git a/collectors/ciscomeraki/local/env.json.tmpl b/collectors/ciscomeraki/local/env.json.tmpl new file mode 100644 index 00000000..b33666f9 --- /dev/null +++ b/collectors/ciscomeraki/local/env.json.tmpl @@ -0,0 +1,17 @@ +{ + "LocalLambda": { + "aims_secret_key": "aims-secret-key-ecnrypted-with-kms-key-from-sam-template", + "aims_access_key_id": "aims-key-id", + "al_api": "api.product.dev.alertlogic.com", + "stack_name": "some-stack", + "azollect_api": "api.product.dev.alertlogic.com", + "ingest_api": "api.global-integration.us-west-2.product.dev.alertlogic.com", + "DEBUG": "*", + "paws_state_queue_arn" :"arn:aws:sqs:us-east-1:352283894008:paws-state-queue", + "paws_state_queue_url" :"https://sqs.us-east-1.amazonaws.com/352283894008/paws-state-queue", + "paws_poll_interval": "900", + "paws_extension": "ciscomeraki", + "sample-custom-var": "https://some-endpoint.com/", + "collector_id": "557D90CB-0BA2-40EC-8A9D-94184612C084" + } +} diff --git a/collectors/ciscomeraki/local/events/event.json b/collectors/ciscomeraki/local/events/event.json new file mode 100644 index 00000000..faafcf55 --- /dev/null +++ b/collectors/ciscomeraki/local/events/event.json @@ -0,0 +1,16 @@ +{ + "RequestType": "Create", + "ServiceToken": "arn:aws:lambda:eu-west-1:352283894008:function:username-test-remov-GetEndpointsLambdaFuncti-RVS9Y1YR1GJR", + "ResponseURL": "https://cloudformation-custom-resource-response-euwest1.s3-eu-west-1.amazonaws.com", + "StackId": "arn:aws:cloudformation:eu-west-1:352283894008:stack/username-test-removed-creds-4/b8d29ba0-c499-11e7-a296-503abe701cd1", + "RequestId": "f0e0cab3-b258-4058-b56a-820c76ff30a5", + "LogicalResourceId": "EndpointAPIs", + "ResourceType": "AWS::CloudFormation::CustomResource", + "ResourceProperties": { + "ServiceToken": "arn:aws:lambda:eu-west-1:352283894008:function:username-test-remov-GetEndpointsLambdaFuncti-RVS9Y1YR1GJR", + "StackName": "username-test-1", + "AwsAccountId": "352283894008", + "VpcId" : "vpc-id", + "LogGroup" : "log-group-name" + } +} diff --git a/collectors/ciscomeraki/local/events/event_poll.json b/collectors/ciscomeraki/local/events/event_poll.json new file mode 100644 index 00000000..ea51f251 --- /dev/null +++ b/collectors/ciscomeraki/local/events/event_poll.json @@ -0,0 +1,20 @@ +{ + "Records": [ + { + "messageId": "f1758a41-f1cb-40a3-957e-546935a15bfc", + "receiptHandle": "AQEB78LmKaaZ+uj8DVnc/O2zQcAe+qKSi3ZGTSBFssIRSmpwwzUEi2vhPTaIBnhOoh0aoRxVjWdoXO3ZloINfMxmcjycP3KC0WXwcWokoOc3iMCdqhYg0NcOhQW1X0ixc79C9/5/XF1xGd79vLhFL7KvRjjiT4sOaSxlAv6v2fJ5eDETnp7CRa5pocCF4EO2su0M4/TnlLreGfsY+C+/tH+r19AM+d3Jt5dbNMrKMWRRZ7/PTjczkIM7U38AHPuusuBz9uzA5yMQGOMI8FPXfqgcafEN17JqKuNSd/l54v1+s9rBQSzL/MH9wZ0XpZditcpe5pTc+dGuRHOXbFK2A0YUQCvRq1Ed8tfr68uELTQK5jWHH/zPnEMWsTyco9VuNCKr", + "body": "{\n \"priv_collector_state\": {\n \n \"since\": \"2024-04-18T16:15:00.000Z\",\n \"until\": \"2024-04-18T16:35:42.740Z\",\n \"nextPage\": null,\n \"networkId\": \"L_686235993220604684\",\n \"poll_interval_sec\": 60,\n \"apiQuotaResetDate\": \"2023-07-07T06:27:42.740Z\" }\n}", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1574159909907", + "SenderId": "some-id", + "ApproximateFirstReceiveTimestamp": "1574159909968" + }, + "messageAttributes": {}, + "md5OfBody": "1845296051d4cfe2e6175358a065cc38", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-east-1:352283894008:paws-state-queue", + "awsRegion": "us-east-1" + } + ] +} diff --git a/collectors/ciscomeraki/local/events/event_register.json b/collectors/ciscomeraki/local/events/event_register.json new file mode 100644 index 00000000..a9410b18 --- /dev/null +++ b/collectors/ciscomeraki/local/events/event_register.json @@ -0,0 +1,16 @@ +{ + "RequestType": "Create", + "ServiceToken": "arn:aws:lambda:eu-west-1:352283894008:function:username-test-remov-GetEndpointsLambdaFuncti-RVS9Y1YR1GJR", + "ResponseURL": "https://cloudformation-custom-resource-response-euwest1.s3-eu-west-1.amazonaws.com/", + "StackId": "arn:aws:cloudformation:eu-west-1:352283894008:stack/username-test-removed-creds-4/b8d29ba0-c499-11e7-a296-503abe701cd1", + "RequestId": "f0e0cab3-b258-4058-b56a-820c76ff30a5", + "LogicalResourceId": "EndpointAPIs", + "ResourceType": "AWS::CloudFormation::CustomResource", + "ResourceProperties": { + "ServiceToken": "arn:aws:lambda:eu-west-1:352283894008:function:username-test-remov-GetEndpointsLambdaFuncti-RVS9Y1YR1GJR", + "StackName": "username-test-1", + "AwsAccountId": "352283894008", + "VpcId" : "vpc-id", + "LogGroup" : "log-group-name" + } +} diff --git a/collectors/ciscomeraki/local/events/event_selfupdate.json b/collectors/ciscomeraki/local/events/event_selfupdate.json new file mode 100644 index 00000000..d83226b2 --- /dev/null +++ b/collectors/ciscomeraki/local/events/event_selfupdate.json @@ -0,0 +1,4 @@ +{ + "RequestType": "ScheduledEvent", + "Type": "SelfUpdate" +} \ No newline at end of file diff --git a/collectors/ciscomeraki/local/run-sam.sh b/collectors/ciscomeraki/local/run-sam.sh new file mode 100755 index 00000000..60f8f082 --- /dev/null +++ b/collectors/ciscomeraki/local/run-sam.sh @@ -0,0 +1,51 @@ +#!/bin/bash +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +SAM_TEMPLATE_NAME="sam-template.yaml" +ENV_FILE_NAME="env.json" +EVENT_FILE_NAME="event_poll.json" +SRC_SAM_TEMPLATE="${SCRIPT_DIR}/sam-template.yaml" +SRC_ENV_FILE="${SCRIPT_DIR}/${ENV_FILE_NAME}" +SRC_EVENT_FILE="${SCRIPT_DIR}/events/${EVENT_FILE_NAME}" +RUN_DIR=${SCRIPT_DIR}/../ +PROFILE_NAME="" + +exists(){ + command -v "$1" >/dev/null 2>&1 +} + +if exists jq; then + uid=`uuidgen` + LOWERUUID=$(echo "$uid" | tr '[:upper:]' '[:lower:]') + echo "generating messageId in event.json: ${LOWERUUID}" + jq --arg newRandomvalue $LOWERUUID '(.Records[].messageId) |= $newRandomvalue' ${SRC_EVENT_FILE} > tmp && mv tmp ${SRC_EVENT_FILE} +else + echo "jq does not exist please install jq to run command" +fi + +command -v sam > /dev/null +if [ $? -ne 0 ]; then + echo "sam not found. +Please follow the installation instructions https://docs.aws.amazon.com/lambda/latest/dg/sam-cli-requirements.html" + exit 0 +fi + +if [ ! -f ${SRC_ENV_FILE} ]; then + echo "${SRC_ENV_FILE} doesn't exist. Please copy and fill in ${SRC_ENV_FILE}.tmpl" + exit 0 +fi + +ln -sf ${SRC_SAM_TEMPLATE} ${RUN_DIR}/${SAM_TEMPLATE_NAME} +ln -sf ${SRC_ENV_FILE} ${RUN_DIR}/${ENV_FILE_NAME} +ln -sf ${SRC_EVENT_FILE} ${RUN_DIR}/${EVENT_FILE_NAME} +cd ${RUN_DIR} && \ +sam local invoke \ + --profile ${PROFILE_NAME} \ + --env-vars ${ENV_FILE_NAME} \ + -t ${SAM_TEMPLATE_NAME}\ + -e ${EVENT_FILE_NAME}\ + --region us-east-1\ + "LocalLambda" + +unlink ${RUN_DIR}/${SAM_TEMPLATE_NAME} +unlink ${RUN_DIR}/${ENV_FILE_NAME} +unlink ${RUN_DIR}/${EVENT_FILE_NAME} diff --git a/collectors/ciscomeraki/local/sam-template.yaml b/collectors/ciscomeraki/local/sam-template.yaml new file mode 100644 index 00000000..802e046c --- /dev/null +++ b/collectors/ciscomeraki/local/sam-template.yaml @@ -0,0 +1,48 @@ +AWSTemplateFormatVersion : '2010-09-09' +Transform: AWS::Serverless-2016-10-31 + +Description: Run it locally +Resources: + LocalLambda: + Type: AWS::Serverless::Function + Properties: + KmsKeyArn: arn:aws:kms:us-east-1:352283894008:key/c31cd559-a589-417b-91bb-19bfbcba903f + Environment: + Variables: + AWS_LAMBDA_FUNCTION_NAME: + aims_secret_key: + LOG_LEVEL: + # DEBUG: + aims_access_key_id: + al_api: + stack_name: + azcollect_api: + ingest_api: + collector_status_api: + paws_state_queue_arn: + paws_state_queue_url: + paws_poll_interval: + paws_extension: + paws_collector_param_string_1: + paws_collector_param_string_2: + paws_api_client_id: + collector_streams: + paws_type_name: "ciscomeraki" + paws_max_pages_per_invocation: + paws_secret_param_name: + paws_api_secret: + collector_id: + customer_id: + paws_endpoint: + paws_ddb_table_name: + ssm_direct: + collector_status_api: + paws_poll_interval_delay: + secret: + dl_s3_bucket_name: + CodeUri: + Runtime: nodejs18.x + Handler: index.handler + Timeout: 300 + MemorySize: 1024 + diff --git a/collectors/ciscomeraki/package.json b/collectors/ciscomeraki/package.json new file mode 100644 index 00000000..a87c75d5 --- /dev/null +++ b/collectors/ciscomeraki/package.json @@ -0,0 +1,36 @@ +{ + "name": "ciscomeraki-collector", + "version": "1.0.0", + "description": "Alert Logic AWS based Ciscomeraki Log Collector", + "repository": {}, + "private": true, + "scripts": { + "lint": "jshint --exclude \"./node_modules/*\" **/*.js", + "test": "JUNIT_REPORT_PATH=./test/report.xml nyc --reporter=text mocha --colors" + }, + "devDependencies": { + "@aws-sdk/client-cloudformation": "^3.454.0", + "@aws-sdk/client-cloudwatch": "^3.454.0", + "@aws-sdk/client-dynamodb": "^3.454.0", + "@aws-sdk/client-kms": "^3.454.0", + "@aws-sdk/client-lambda": "^3.454.0", + "@aws-sdk/client-s3": "^3.456.0", + "@aws-sdk/client-sqs": "^3.454.0", + "@aws-sdk/client-ssm": "^3.454.0", + "jshint": "^2.9.5", + "mocha": "^10.2.0", + "mocha-jenkins-reporter": "^0.4.2", + "nyc": "^15.1.0", + "rewire": "^7.0.0", + "sinon": "^17.0.0" + }, + "dependencies": { + "@alertlogic/al-collector-js": "3.0.10", + "@alertlogic/paws-collector": "2.2.1", + "async": "^3.2.4", + "debug": "^4.3.4", + "moment": "2.29.4" + + }, + "author": "Alert Logic Inc." +} \ No newline at end of file diff --git a/collectors/ciscomeraki/test/ciscomeraki_mock.js b/collectors/ciscomeraki/test/ciscomeraki_mock.js new file mode 100644 index 00000000..0a728922 --- /dev/null +++ b/collectors/ciscomeraki/test/ciscomeraki_mock.js @@ -0,0 +1,104 @@ +process.env.AWS_REGION = 'us-east-1'; +process.env.al_api = 'api.global-services.global.alertlogic.com'; +process.env.ingest_api = 'ingest.global-services.global.alertlogic.com'; +process.env.azollect_api = 'azcollect.global-services.global.alertlogic.com'; +process.env.collector_status_api = 'collector_status.global-services.global.alertlogic.com'; + +process.env.aims_access_key_id = 'aims-key-id'; +process.env.aims_secret_key = 'aims-secret-key-encrypted'; +process.env.log_group = 'logGroupName'; +process.env.paws_state_queue_arn = "arn:aws:sqs:us-east-1:352283894008:paws-state-queue"; +process.env.paws_extension = 'ciscomeraki'; +process.env.ciscomeraki_endpoint = 'https://test.alertlogic.com/'; +process.env.ciscomeraki_token = 'ciscomeraki-token'; +process.env.collector_id = 'collector-id'; +process.env.al_application_id = 'application_id'; +process.env.paws_secret_param_name = "ciscomeraki-param-name"; +process.env.paws_poll_interval = 60; +process.env.paws_type_name = "ciscomeraki"; +process.env.paws_collector_param_string_2 = "12345"; +process.env.paws_collector_param_string_1 = "[\"appliance\",\"systemsManager\",\"switch\"]"; +process.env.paws_api_secret = "secret"; +process.env.paws_endpoint = "api.meraki.com"; +process.env.collector_streams = "[\"L_686235993220604684\"]"; +process.env.paws_api_client_id = "client-id"; +process.env.paws_poll_interval_delay = 300; + +const AIMS_TEST_CREDS = { + access_key_id: 'test-access-key-id', + secret_key: 'test-secret-key' +}; + +const LOG_EVENT = { + "occurredAt": "2024-03-19T05:10:47.055027Z", + "networkId": "L_686235993220604684", + "type": "client_vpn", + "description": "Client VPN negotiation", + "clientId": null, + "clientDescription": null, + "clientMac": "", + "category": "wired_only_client_vpn", + "deviceSerial": "Q3FA-KM7T-NYGZ", + "deviceName": "", + "eventData": { + "msg": " deleting IKE_SA l2tp-over-ipsec-1[97] between 209.163.151.90[209.163.151.90]...117.200.14.68[192.168.1.6]" + } + }; + + const NETWORKS = [ + { + id: 'L_686235993220604684', + organizationId: '1547127', + name: 'Alert Logic Test Kit', + productTypes: [ + 'appliance', + 'camera', + 'cellularGateway', + 'sensor', + 'switch', + 'wireless' + ], + timeZone: 'America/Los_Angeles', + tags: [], + enrollmentString: null, + url: 'https://n219.meraki.com/Alert-Logic-Test/n/el2PiaBd/manage/usage/list', + notes: 'Test node', + isBoundToConfigTemplate: false + }, + { + id: 'L_686235993220604720', + organizationId: '1547127', + name: 'Alert Logic Test Kit 1 ', + productTypes: [ + 'appliance', + 'camera', + 'cellularGateway', + 'sensor', + 'switch', + 'wireless' + ], + timeZone: 'America/Los_Angeles', + tags: [], + enrollmentString: null, + url: 'https://n219.meraki.com/Alert-Logic-Test/n/yNvOHaBd/manage/usage/list', + notes: 'Test node', + isBoundToConfigTemplate: false + } + ]; + +const mockInitialStates = [ + { networkId: 'L_686235993220604684', since: '2024-03-20T07:24:34.657Z', until: '2024-03-20T07:25:34.657Z', nextPage: null }, + { networkId: 'L_686235993220604682', since: '2024-03-20T07:24:34.657Z', until: '2024-03-20T07:25:34.657Z', nextPage: null } +]; + +const FUNCTION_ARN = 'arn:aws:lambda:us-east-1:352283894008:function:test-01-CollectLambdaFunction-2CWNLPPW5XO8'; +const FUNCTION_NAME = 'test-TestCollectLambdaFunction-1JNNKQIPOTEST'; + +module.exports = { + AIMS_TEST_CREDS: AIMS_TEST_CREDS, + FUNCTION_ARN: FUNCTION_ARN, + FUNCTION_NAME: FUNCTION_NAME, + LOG_EVENT: LOG_EVENT, + NETWORKS:NETWORKS, + mockInitialStates:mockInitialStates +}; diff --git a/collectors/ciscomeraki/test/ciscomeraki_test.js b/collectors/ciscomeraki/test/ciscomeraki_test.js new file mode 100644 index 00000000..a1819199 --- /dev/null +++ b/collectors/ciscomeraki/test/ciscomeraki_test.js @@ -0,0 +1,829 @@ +const sinon = require('sinon'); +const assert = require('assert'); + +const ciscomerakiMock = require('./ciscomeraki_mock'); +var CiscomerakiCollector = require('../collector').CiscomerakiCollector; +const m_response = require('cfn-response'); +const moment = require('moment'); +const utils = require("../utils"); + +const { CloudWatch } = require("@aws-sdk/client-cloudwatch"), + { KMS } = require("@aws-sdk/client-kms"), + { SSM } = require("@aws-sdk/client-ssm"); + +var responseStub = {}; +let getAPIDetails; +let getAPILogs; +let getAllNetworks; + +describe('Unit Tests', function () { + beforeEach(function () { + sinon.stub(SSM.prototype, 'getParameter').callsFake(function (params, callback) { + const data = Buffer.from('test-secret'); + return callback(null, { Parameter: { Value: data.toString('base64') } }); + }); + sinon.stub(KMS.prototype, 'decrypt').callsFake(function (params, callback) { + const data = { + Plaintext: Buffer.from('{}') + }; + return callback(null, data); + }); + + responseStub = sinon.stub(m_response, 'send').callsFake( + function fakeFn(event, mockContext, responseStatus, responseData, physicalResourceId) { + mockContext.succeed(); + }); + }); + + afterEach(function () { + responseStub.restore(); + KMS.prototype.decrypt.restore(); + SSM.prototype.getParameter.restore(); + }); + + describe('Paws Get Register Parameters', function () { + it('Paws Get Register Parameters Success', function (done) { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const sampleEvent = { ResourceProperties: { StackName: 'a-stack-name' } }; + collector.pawsGetRegisterParameters(sampleEvent, (err, regValues) => { + const expectedRegValues = { + ciscoMerakiObjectNames: process.env.collector_streams, + }; + assert.deepEqual(regValues, expectedRegValues); + done(); + }); + }); + }); + }); + + describe('Paws Init Collection State', function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + it('Paws Init Collection State Success', function (done) { + getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return resolve(ciscomerakiMock.NETWORKS); + }); + }); + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment().subtract(1, 'days').toISOString(); + process.env.paws_collection_start_ts = startDate; + + collector.pawsInitCollectionState(null, (err, initialStates, nextPoll) => { + initialStates.forEach((state) => { + if (state.networkId === "L_686235993220604684") { + assert.equal(state.networkId, "L_686235993220604684"); + } else if (state.networkId === "L_686235993220604720") { + assert.equal(state.networkId, "L_686235993220604720"); + } + else { + assert.equal(state.poll_interval_sec, 1); + assert.ok(state.since); + } + }); + }); + }); + done(); + }); + it('Paws Init Collection State with networks', function (done) { + process.env.collector_streams = []; + // Mocking utils.getAllNetworks to return a non-empty array + getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + function fakeFn(callback) { + return new Promise(function (resolve, reject) { + resolve(ciscomerakiMock.NETWORKS); + }); + }); + + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment().subtract(1, 'days').toISOString(); + process.env.paws_collection_start_ts = startDate; + + collector.pawsInitCollectionState(null, (err, initialStates, nextPoll) => { + // Asserting that initialStates are generated correctly + assert.equal(initialStates.length, ciscomerakiMock.NETWORKS.length); + initialStates.forEach((state) => { + assert.ok(state.networkId); // Assuming networkId exists in each state + }); + done(); + }); + }); + }); + + it('Paws Init Collection State without networks', function (done) { + process.env.collector_streams = []; + getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + function fakeFn(callback) { + return new Promise(function (resolve, reject) { + resolve([]); // Assuming no networks found + }); + }); + + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment().subtract(1, 'days').toISOString(); + process.env.paws_collection_start_ts = startDate; + + collector.pawsInitCollectionState(null, (err, initialStates, nextPoll) => { + // Asserting that an error is returned when no networks are found + assert.equal(err, "Error: No networks found"); + done(); + }); + }); + }); + + it('Paws Init Collection State error handling', function (done) { + getAllNetworks = sinon.stub(utils, 'getAllNetworks').rejects(new Error('Network error')); + + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment().subtract(1, 'days').toISOString(); + process.env.paws_collection_start_ts = startDate; + + collector.pawsInitCollectionState(null, (err, initialStates, nextPoll) => { + assert.equal(err.message, 'Network error'); + done(); + }); + }); + }); + + afterEach(function () { + getAllNetworks.restore(); + }); + }); + + describe('pawsGetLogs', function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + it('Paws Get Logs Success', function (done) { + getAPILogs = sinon.stub(utils, 'getAPILogs').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return resolve({ accumulator: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT] }); + }); + }); + getAPIDetails = sinon.stub(utils, 'getAPIDetails').callsFake( + function fakeFn(state) { + return { + url: "api_url", + method: "GET", + requestBody: "", + orgKey: "1234", + productTypes: ["appliance"] + + }; + }); + getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return resolve(ciscomerakiMock.NETWORKS); + }); + }); + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const curState = { + networkId: "L_686235993220604684", + since: "2024-03-19T05:10:47.055027Z", + until: null, + nextPage: null, + poll_interval_sec: 1 + }; + collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) => { + assert.equal(logs.length, 2); + assert.equal(newState.poll_interval_sec, 300); + assert.ok(logs[0].type); + getAPILogs.restore(); + getAllNetworks.restore(); + getAPIDetails.restore(); + }); + + done(); + + }); + }); + + it('Paws Get Logs With NextPage Success', function (done) { + getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return resolve(ciscomerakiMock.NETWORKS); + }); + }); + getAPILogs = sinon.stub(utils, 'getAPILogs').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return resolve({ accumulator: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], nextPage: "nextPage" }); + }); + }); + getAPIDetails = sinon.stub(utils, 'getAPIDetails').callsFake( + function fakeFn(state) { + return { + url: "api_url", + method: "GET", + requestBody: "", + orgKey: "1234", + productTypes: ["appliance"] + + }; + }); + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = "2024-03-21T08:00:21.754Z"; + const curState = { + networkId: "L_686235993220604684", + since: startDate.valueOf(), + nextPage: null, + poll_interval_sec: 1 + }; + collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) => { + assert.equal(logs.length, 2); + assert.equal(newState.poll_interval_sec, 1); + assert.equal(newState.nextPage, null); + assert.equal(newState.since, 'nextPage'); + assert.ok(logs[0].type); + getAPILogs.restore(); + getAPIDetails.restore(); + getAllNetworks.restore(); + + done(); + }); + + }); + }); + + it('Paws Get client error', function (done) { + + getAPILogs = sinon.stub(utils, 'getAPILogs').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return reject({ + code: 401, + message: 'Invalid API key', + response: { + data: { errors: ['Invalid API key'] }, + status: 401 + } + }); + }); + }); + getAPIDetails = sinon.stub(utils, 'getAPIDetails').callsFake( + function fakeFn(state) { + return { + url: "api_url", + method: "GET", + requestBody: "", + orgKey: "1234", + productTypes: ["appliance"] + + }; + }); + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment(); + const curState = { + networkId: "L_686235993220604684", + since: startDate.valueOf(), + nextPage: null, + poll_interval_sec: 1 + }; + collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) => { + assert.equal(err.errorCode, 401); + getAPILogs.restore(); + getAPIDetails.restore(); + done(); + }); + + }); + }); + + it('Paws Get Logs check throttling error', function (done) { + + getAPILogs = sinon.stub(utils, 'getAPILogs').callsFake( + function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + return new Promise(function (resolve, reject) { + return reject({ + code: 429, + message: 'Too Many Requests', + response: { + data: { errors: ['Too Many Requests'] }, + headers: { 'retry-after': 360 }, + status: 429 + }, + stat: 'FAIL', "errorCode": 429 + }); + }); + }); + getAPIDetails = sinon.stub(utils, 'getAPIDetails').callsFake( + function fakeFn(state) { + return { + url: "api_url", + method: "GET", + requestBody: "", + orgKey: "1234", + productTypes: ["appliance"] + + }; + }); + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment(); + const curState = { + networkId: "L_686235993220604684", + since: startDate.valueOf(), + nextPage: null, + poll_interval_sec: 1 + }; + + var reportSpy = sinon.spy(collector, 'reportApiThrottling'); + let putMetricDataStub = sinon.stub(CloudWatch.prototype, 'putMetricData').callsFake((params, callback) => callback()); + collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) => { + assert.equal(true, reportSpy.calledOnce); + assert.equal(logs.length, 0); + assert.notEqual(newState.poll_interval_sec, 1); + getAPILogs.restore(); + getAPIDetails.restore(); + putMetricDataStub.restore(); + done(); + }); + + }); + }); + // it('Handles successful API log retrieval with logs and next page', function (done) { + // getAPILogs = sinon.stub(utils, 'getAPILogs').callsFake( + // function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + // return new Promise(function (resolve, reject) { + // return resolve({ accumulator: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], nextPage: "nextPage" }); + // }); + // }); + // getAPIDetails = sinon.stub(utils, 'getAPIDetails').callsFake( + // function fakeFn(state) { + // return { + // url: "api_url", + // method: "GET", + // requestBody:"", + // orgKey:"1234", + // productTypes:["appliance"] + + // }; + // }); + // getAllNetworks = sinon.stub(utils, 'getAllNetworks').callsFake( + // function fakeFn(client, objectDetails, state, accumulator, maxPagesPerInvocation) { + // return new Promise(function (resolve, reject) { + // return resolve(ciscomerakiMock.NETWORKS); + // }); + // }); + // CiscomerakiCollector.load().then(function (creds) { + // var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + // const curState = { + // networkId: "L_686235993220604684", + // since: "2024-03-19T05:10:47.055027Z", + // until: null, + // nextPage: null, + // poll_interval_sec: 1 + // }; + // collector.pawsGetLogs(curState, (err, logs, newState, newPollInterval) => { + // assert.equal(logs.length, 2); + // assert.equal(newState.poll_interval_sec, 300); + // assert.ok(logs[0].type); + // assert.equal(newState.nextPage, "nextPage"); + // getAPILogs.restore(); + // getAPIDetails.restore(); + // getAllNetworks.restore(); + // done(); + // }); + // }); + // }); + }); + + describe('Next state tests', function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + + it('Next state tests success with L_686235993220604684', function (done) { + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const startDate = moment(); + const curState = { + networkId: "L_686235993220604684", + since: startDate.valueOf(), + until: startDate.add(collector.pollInterval, 'seconds').valueOf(), + nextPage: null, + poll_interval_sec: 1 + }; + let nextState = collector._getNextCollectionState(curState); + assert.equal(nextState.poll_interval_sec, process.env.paws_poll_interval_delay); + done(); + }); + }); + + }); + + describe('Format Tests', function () { + it('log format success', function (done) { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + done(); + }, + succeed: function () { + done(); + } + }; + + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + let fmt = collector.pawsFormatLog(ciscomerakiMock.LOG_EVENT); + assert.equal(fmt.progName, 'CiscomerakiCollector'); + assert.ok(fmt.message); + done(); + }); + }); + }); + + describe('NextCollectionStateWithNextPage', function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + it('Get Next Collection State (L_686235993220604684) With NextPage Success', function (done) { + const startDate = moment(); + const curState = { + networkId: "L_686235993220604684", + since: startDate.valueOf(), + poll_interval_sec: 1 + }; + const nextPage = "nextPage"; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + let nextState = collector._getNextCollectionStateWithNextPage(curState, nextPage); + assert.ok(nextState.since); + assert.equal(nextState.since, nextPage); + done(); + }); + }); + it('Get Next Collection State (L_686235993220604684) With NextPage Success', function (done) { + const startDate = moment(); + const curState = { + networkId: "L_686235993220604684", + since: startDate.unix(), + poll_interval_sec: 1 + }; + const nextPageTimestamp = "1574157600"; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + let nextState = collector._getNextCollectionStateWithNextPage(curState, nextPageTimestamp); + assert.ok(nextState.since); + assert.equal(nextState.since, nextPageTimestamp); + done(); + }); + }); + }); + describe('Next Collection State Calculation', function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + it('should calculate the next collection state correctly', function (done) { + const curState = { + networkId: "L_686235993220604684", + since: moment().valueOf(), + poll_interval_sec: 1 + }; + + const expectedNextState = { + networkId: 'L_686235993220604684', + since: moment().toISOString(), + until: moment().add(1, 'minutes').toISOString(), + nextPage: null, + poll_interval_sec: '300' + }; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + let nextState = collector._getNextCollectionState(curState); + assert.deepEqual(nextState.poll_interval_sec, expectedNextState.poll_interval_sec); + done(); + }); + }); + + it('handles ScheduledEvent with SelfUpdate', function (done) { + let updateNetworksStub; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const event = { RequestType: 'ScheduledEvent', Type: 'SelfUpdate' }; + updateNetworksStub = sinon.stub(collector, 'handleUpdateStreamsFromNetworks'); + collector.handleEvent(event); + assert(updateNetworksStub.calledOnce); + updateNetworksStub.restore(); + done(); + }); + }); + it('handles other event types', () => { + + let updateNetworksStub; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + const event = { RequestType: 'OtherEventType' }; + collector.handleEvent(event); + assert(!updateNetworksStub.called); + updateNetworksStub.restore(); + }); + + }); + it('updates networks', () => { + let updateNetworksStub; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + updateNetworksStub.returns(Promise.resolve()); + collector.handleScheduledEvent({ Type: 'SelfUpdate' }); + assert(updateNetworksStub.calledOnce); + updateNetworksStub.restore(); + }); + }); + + }); + + describe('Error Handling', function () { + describe('handleThrottlingError', function () { + it('should retry after adding random time to poll_interval_sec', function () { + const error = { + response: { + headers: { + 'retry-after': '10' + } + } + }; + const state = { + poll_interval_sec: 1 + }; + const callback = sinon.stub(); + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error1) { + assert.fail(error1); + }, + succeed: function () { } + }; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + collector.handleThrottlingError(error, state, callback); + + assert.strictEqual(callback.calledOnce, true); + assert.strictEqual(typeof callback.args[0][0], 'object'); + assert.strictEqual(callback.args[0][1].length, 0); + assert.strictEqual(callback.args[0][2], state); + assert.strictEqual(typeof callback.args[0][3], 'number'); + }); + }); + }); + + describe('handleOtherErrors', function () { + it('should retry if API_NOT_FOUND_ERROR occurs less than 3 times', function () { + const error = { + response: { + status: CiscomerakiCollector.API_NOT_FOUND_ERROR + } + }; + const state = { + retry: 2 + }; + const callback = sinon.stub(); + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error1) { + assert.fail(error1); + }, + succeed: function () { } + }; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + collector.handleOtherErrors(error, state, callback); + + assert.strictEqual(callback.calledOnce, false); + assert.strictEqual(state.retry, 3); + }); + }); + + it('should succeed if API_NOT_FOUND_ERROR occurs 3 times', function () { + const error = { + response: { + status: CiscomerakiCollector.API_NOT_FOUND_ERROR + } + }; + const state = { + retry: 3 + }; + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error1) { + assert.fail(error1); + }, + succeed: function () { } + }; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + const succeedStub = sinon.stub(collector._invokeContext, 'succeed'); + + collector.handleOtherErrors(error, state, () => { }); + + assert.strictEqual(succeedStub.calledOnce, true); + }); + }); + + it('should return error if response has data', function () { + const error = { + response: { + data: { + errors: ['Invalid API key'] + }, + status: 401 + } + }; + const state = {}; + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error1) { + assert.fail(error1); + }, + succeed: function () { } + }; + CiscomerakiCollector.load().then(function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + const callbackStub = sinon.stub(); + + collector.handleOtherErrors(error, state, callbackStub); + + assert.strictEqual(callbackStub.calledOnce, true); + assert.deepStrictEqual(callbackStub.args[0][0], { + errors: ['Invalid API key'], + errorCode: 401 + }); + }); + }); + + it('should return original error if response has no data', function () { + const error = { + response: { + status: 500 + } + }; + const state = {}; + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error1) { + assert.fail(error1); + }, + succeed: function () { } + }; + CiscomerakiCollector.load().then(function (creds) { + + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + const callbackStub = sinon.stub(); + + collector.handleOtherErrors(error, state, callbackStub); + + assert.strictEqual(callbackStub.calledOnce, true); + assert.strictEqual(callbackStub.args[0][0], error); + }); + }); + }); + }); + +}); + +describe('handleUpdateStreamsFromNetworks Function', function () { + let uploadStub; + beforeEach(function () { + sinon.stub(utils, 'getAllNetworks').resolves(['network1', 'network2']); + sinon.stub(utils, 'getS3ObjectParams').resolves({ bucketName: 'testBucket', key: 'testKey' }); + sinon.stub(utils, 'fetchJsonFromS3Bucket').resolves(['network1']); + sinon.stub(utils, 'differenceOfNetworksArray').returns(['network2']); + uploadStub = sinon.stub(utils, 'uploadNetworksListInS3Bucket').resolves(); + }); + + afterEach(function () { + sinon.restore(); + }); + + it('should handle network updates correctly', async function () { + + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + + return CiscomerakiCollector.load().then(async function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + + sinon.stub(collector, '_storeCollectionState').callsFake(function (_, __, ___, callback) { + callback(); + }); + + await collector.handleUpdateStreamsFromNetworks(); + + assert.strictEqual(utils.getAllNetworks.callCount, 1); + assert.strictEqual(utils.getS3ObjectParams.callCount, 1); + assert.strictEqual(utils.fetchJsonFromS3Bucket.callCount, 1); + assert.strictEqual(utils.differenceOfNetworksArray.callCount, 1); + assert.strictEqual(uploadStub.callCount, 1); + sinon.restore(); + }); + }); + + it('should handle network updates when no networks from S3 are found', async function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + + return CiscomerakiCollector.load().then(async function (creds) { + const collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + sinon.stub(collector, '_storeCollectionState').callsFake(function (_, __, ___, callback) { + callback(); + }); + + await collector.handleUpdateStreamsFromNetworks(); + + assert.strictEqual(utils.getAllNetworks.callCount, 1); + assert.strictEqual(utils.getS3ObjectParams.callCount, 1); + assert.strictEqual(utils.fetchJsonFromS3Bucket.callCount, 1); + assert.strictEqual(uploadStub.callCount, 1); + }); + }); + + it('should handle network updates when S3 fetch returns an error', async function () { + let ctx = { + invokedFunctionArn: ciscomerakiMock.FUNCTION_ARN, + fail: function (error) { + assert.fail(error); + }, + succeed: function () { } + }; + return CiscomerakiCollector.load().then(async function (creds) { + var collector = new CiscomerakiCollector(ctx, creds, 'ciscomeraki'); + sinon.stub(collector, '_storeCollectionState').callsFake(function (_, __, ___, callback) { + callback(); + }); + + await collector.handleUpdateStreamsFromNetworks(); + assert.strictEqual(utils.getAllNetworks.callCount, 1); + assert.strictEqual(utils.getS3ObjectParams.callCount, 1); + assert.strictEqual(utils.fetchJsonFromS3Bucket.callCount, 1); + assert.strictEqual(uploadStub.callCount, 1); + }); + }); +}); + + diff --git a/collectors/ciscomeraki/test/utils_test.js b/collectors/ciscomeraki/test/utils_test.js new file mode 100644 index 00000000..9a2f2038 --- /dev/null +++ b/collectors/ciscomeraki/test/utils_test.js @@ -0,0 +1,442 @@ +const sinon = require('sinon'); +const assert = require('assert'); +const axios = require('axios'); +const ciscomerakiMock = require('./ciscomeraki_mock'); +const AlLogger = require('@alertlogic/al-aws-collector-js').Logger; +const AlAwsUtil = require('@alertlogic/al-aws-collector-js').Util; +const { getAPILogs, makeApiCall, getAllNetworks, fetchAllNetworks, getAPIDetails, + fetchJsonFromS3Bucket, differenceOfNetworksArray, getOrgKeySecretEndPoint, getS3ObjectParams, uploadToS3Bucket } = require('../utils'); +const { S3Client } = require("@aws-sdk/client-s3"); + +describe('API Tests', function () { + let axiosGetStub; + + beforeEach(function () { + axiosGetStub = sinon.stub(axios, 'get'); + }); + + afterEach(function () { + axiosGetStub.restore(); + }); + + describe('getAPILogs', function () { + it('should accumulate data from multiple pages', async function () { + axiosGetStub.onFirstCall().returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], pageEndAt: '2024-04-15T10:00:00Z' }, + headers: { link: '; rel=next' } + })); + axiosGetStub.onSecondCall().returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], pageEndAt: '2024-04-15T11:00:00Z' }, + headers: {} + })); + + const apiDetails = { productTypes: ['appliance', 'switch'] }; + const accumulator = []; + const apiEndpoint = 'api.meraki.com'; + const state = { since: '2024-04-14T00:00:00Z', networkId: 'L_686235993220604684' }; + const clientSecret = 'your-secret'; + const maxPagesPerInvocation = 2; + + const result = await getAPILogs(apiDetails, accumulator, apiEndpoint, state, clientSecret, maxPagesPerInvocation); + + assert.deepStrictEqual(result.accumulator, [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT]); + assert.strictEqual(result.nextPage, undefined); + }); + }); + + describe('makeApiCall', function () { + it('should return response data', async function () { + let url = 'https://api.meraki.com/network/L_686235993220604684/events'; + const apiKey = 'your-api-key'; + const perPage = 10; + const startingAfter = null; + + axiosGetStub.returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT] }, + headers: {} + })); + + const response = await makeApiCall(url, apiKey, perPage, startingAfter); + + assert.deepStrictEqual(response.data.events, [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT]); + }); + }); + + describe('fetchAllNetworks', function () { + it('should return network data', async function () { + const url = '/api/v1/networks'; + const apiKey = 'your-api-key'; + const apiEndpoint = 'api.meraki.com'; + + axiosGetStub.returns(Promise.resolve({ + data: { networks: ['L_686235993220604684', 'L_686235993220604685'] }, + headers: {} + })); + + const networks = await fetchAllNetworks(url, apiKey, apiEndpoint); + + assert.deepStrictEqual(networks.networks, ['L_686235993220604684', 'L_686235993220604685']); + }); + }); + + describe('getAPIDetails', function () { + it('should return correct API details', function () { + const orgKey = 'your-org-key'; + const productTypes = ['appliance', 'switch']; + + const apiDetails = getAPIDetails(orgKey, productTypes); + + assert.strictEqual(apiDetails.url, '/api/v1/networks'); + assert.strictEqual(apiDetails.method, 'GET'); + assert.strictEqual(apiDetails.requestBody, ''); + assert.strictEqual(apiDetails.orgKey, orgKey); + assert.deepStrictEqual(apiDetails.productTypes, productTypes); + }); + }); + + describe('Error Handling', function () { + it('should handle network errors gracefully', async function () { + axiosGetStub.rejects(new Error('Network error')); + try { + await makeApiCall('https://api.meraki.com/network/L_686235993220604684/events', 'your-api-key', 10); + } catch (error) { + assert.equal(error.message, 'Network error'); + } + }); + }); + + describe('Pagination Handling', function () { + it('should correctly handle the last page of results', async function () { + const apiDetails = { productTypes: ['appliance', 'switch'] }; + const apiEndpoint = 'api.meraki.com'; + const state = { since: '2024-04-14T00:00:00Z', networkId: 'L_686235993220604684' }; + const clientSecret = 'your-secret'; + const maxPagesPerInvocation = 2; + // Assuming you have a way to track the number of calls to axios.get + let callCount = 0; + axiosGetStub.callsFake(function () { + callCount++; + if (callCount === 1) { + return Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], pageEndAt: '2024-04-16T10:00:00Z' }, + headers: {} + }); + } else if (callCount === 2) { + return Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], pageEndAt: '2024-04-16T10:00:00Z' }, + headers: {} + }); + } else { + return Promise.resolve({ + data: { events: [] }, + headers: {} + }); + } + }); + + const result = await getAPILogs(apiDetails, [], apiEndpoint, state, clientSecret, maxPagesPerInvocation); + const mockResults = { nextPage: undefined, accumulator: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT] }; + assert.equal(result.accumulator.length, mockResults.accumulator.length); + assert.equal(result.nextPage, mockResults.nextPage); + }); + }); + + describe('Rate Limiting', function () { + it('should apply exponential backoff on rate limit errors', async function () { + axiosGetStub.rejects({ response: { status: 429 } }); + const attemptApiCall = getAllNetworks('/api/v1/networks', 'your-api-key', 'api.meraki.com'); + attemptApiCall.catch((error) => { + assert.include(error.message, '429'); + }); + }); + }); + + describe('API Details Functionality', function () { + it('should construct correct API details for different orgKey and productTypes', function () { + const orgKey1 = 'another-org-key'; + const productTypes1 = ['switch']; + const apiDetails1 = getAPIDetails(orgKey1, productTypes1); + assert.equal(apiDetails1.url, '/api/v1/networks'); + assert.equal(apiDetails1.method, 'GET'); + assert.equal(apiDetails1.requestBody, ''); + assert.equal(apiDetails1.orgKey, orgKey1); + assert.deepEqual(apiDetails1.productTypes, productTypes1); + + const orgKey2 = 'yet-another-org-key'; + const productTypes2 = ['appliance']; + const apiDetails2 = getAPIDetails(orgKey2, productTypes2); + assert.equal(apiDetails2.url, '/api/v1/networks'); + assert.equal(apiDetails2.method, 'GET'); + assert.equal(apiDetails2.requestBody, ''); + assert.equal(apiDetails2.orgKey, orgKey2); + assert.deepEqual(apiDetails2.productTypes, productTypes2); + }); + }); + + describe('fetchAllNetworks Pagination', function () { + it('should paginate through all networks correctly', async function () { + const url = '/api/v1/networks'; + const apiKey = 'your-api-key'; + const apiEndpoint = 'api.meraki.com'; + + axiosGetStub.resolves({ + data: { networks: ['L_686235993220604684', 'L_686235993220604685'], hasNextPage: true }, + headers: { startingAfter: '2024-01-01T00:00:00' } + }); + + const networks = await fetchAllNetworks(url, apiKey, apiEndpoint); + assert.equal(networks.networks.length, 2); + }); + }); + + + + + describe('getAPIDetails Return Values', function () { + it('should return correct API details', function () { + const orgKey = 'your-org-key'; + const productTypes = ['appliance', 'switch']; + + const apiDetails = getAPIDetails(orgKey, productTypes); + + assert.strictEqual(apiDetails.url, '/api/v1/networks'); + assert.strictEqual(apiDetails.method, 'GET'); + assert.strictEqual(apiDetails.requestBody, ''); + assert.strictEqual(apiDetails.orgKey, orgKey); + assert.deepStrictEqual(apiDetails.productTypes, productTypes); + }); + }); + + describe('fetchJsonFromS3Bucket', function () { + it('should fetch JSON data from S3 bucket', async function () { + const bucketName = 'test-bucket'; + const fileName = 'test.json'; + const mockResponseBody = JSON.stringify([{ id: 1, name: "Test" }]); + + const s3SendStub = sinon.stub(S3Client.prototype, 'send').resolves({ + Body: mockResponseBody + }); + + const result = await fetchJsonFromS3Bucket(bucketName, fileName); + + assert.equal(JSON.stringify(result), mockResponseBody); + + s3SendStub.restore(); + }); + }); + + describe('getAPILogs - Max Pages Per Invocation Reached', function () { + it('should stop accumulating logs after maxPagesPerInvocation and return accumulated data', async function () { + const apiDetails = { productTypes: ['appliance', 'switch'] }; + const apiEndpoint = 'api.meraki.com'; + const state = { since: '2024-04-14T00:00:00Z', networkId: 'L_686235993220604684' }; + const clientSecret = 'your-secret'; + const maxPagesPerInvocation = 2; + + axiosGetStub.onFirstCall().returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], startingAfter: '2024-04-15T10:00:00Z' }, + headers: { link: '; rel=next' } + })); + axiosGetStub.onSecondCall().returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], startingAfter: '2024-04-16T10:00:00Z' }, + headers: {} + })); + + const result = await getAPILogs(apiDetails, [], apiEndpoint, state, clientSecret, maxPagesPerInvocation); + + assert.deepStrictEqual(result.accumulator.length, 4); // Total events from both pages + assert.strictEqual(result.nextPage, undefined); // No more pages to fetch + }); + }); + + describe('Error Handling in fetchAllNetworks', function () { + it('should handle non-rate-limit errors gracefully', async function () { + axiosGetStub.rejects({ response: { status: 500 } }); + try { + await fetchAllNetworks('/api/v1/networks', 'your-api-key', 'api.meraki.com'); + } catch (error) { + assert.equal(error.response.status, 500); + } + }); + }); + + describe('differenceOfNetworksArray', function () { + it('should return the difference between two arrays of network IDs', function () { + // Mock data + const newNetworks = ['L_686235993220604684', 'L_686235993220604685', 'L_686235993220604686']; + const oldNetworks = ['L_686235993220604684', 'L_686235993220604686']; + + const difference = differenceOfNetworksArray(newNetworks, oldNetworks); + + assert.deepStrictEqual(difference, ['L_686235993220604685']); + }); + + it('should return an empty array if oldNetworks contains all network IDs from newNetworks', function () { + const newNetworks = ['L_686235993220604684', 'L_686235993220604685']; + const oldNetworks = ['L_686235993220604684', 'L_686235993220604685', 'L_686235993220604686']; + + const difference = differenceOfNetworksArray(newNetworks, oldNetworks); + + assert.deepStrictEqual(difference, []); + }); + + it('should return newNetworks if oldNetworks is empty', function () { + // Mock data + const newNetworks = ['L_686235993220604684', 'L_686235993220604685']; + + const difference = differenceOfNetworksArray(newNetworks, []); + + assert.deepStrictEqual(difference, newNetworks); + }); + + it('should return an empty array if both newNetworks and oldNetworks are empty', function () { + const difference = differenceOfNetworksArray([], []); + + assert.deepStrictEqual(difference, []); + }); + }); + + describe('Error Handling in getAPILogs', function () { + it('should throw an error when networkId is not provided', async function () { + const apiDetails = { productTypes: ['appliance', 'switch'] }; + const accumulator = []; + const apiEndpoint = 'api.meraki.com'; + const state = { since: '2024-04-14T00:00:00Z' }; + const clientSecret = 'your-secret'; + const maxPagesPerInvocation = 2; + + axiosGetStub.returns(Promise.resolve({ + data: { events: [ciscomerakiMock.LOG_EVENT, ciscomerakiMock.LOG_EVENT], pageEndAt: '2024-04-15T10:00:00Z' }, + headers: { link: '; rel=next' } + })); + + try { + await getAPILogs(apiDetails, accumulator, apiEndpoint, state, clientSecret, maxPagesPerInvocation); + assert.fail('Expected an error to be thrown'); + } catch (error) { + assert.equal(error.message, 'url is not defined'); + } + }); + }); + + describe('getOrgKeySecretEndPoint', function () { + it('should return clientSecret, apiEndpoint, and orgKey when all parameters are provided', function () { + process.env.paws_endpoint = 'https://meraki.com'; + process.env.paws_collector_param_string_2 = 'orgKey123'; + + const result = getOrgKeySecretEndPoint('mockSecret'); + + assert.equal(result.clientSecret, 'mockSecret'); + assert.equal(result.apiEndpoint, 'meraki.com'); + assert.equal(result.orgKey, 'orgKey123'); + }); + + it('should call callback with error message if clientSecret is missing', function () { + process.env.paws_endpoint = 'https://example.com'; + process.env.paws_collector_param_string_2 = 'orgKey123'; + const result = getOrgKeySecretEndPoint(null); + assert.strictEqual(result, 'The Client Secret was not found!'); + }); + + it('should call callback with error message if orgKey is missing', function () { + process.env.paws_endpoint = 'https://example.com'; + process.env.paws_collector_param_string_2 = ''; + const result = getOrgKeySecretEndPoint('mockSecret'); + + assert.strictEqual(result, 'orgKey was not found!'); + }); + }); + + describe('getS3ObjectParams', function () { + it('should return correct params', async function () { + process.env.dl_s3_bucket_name = 'test-bucket'; + const keyValue = 'test-key'; + const data = { key: 'value' }; + + const params = await getS3ObjectParams(keyValue, data); + + assert.deepStrictEqual(params, { + data: data, + key: keyValue, + bucketName: 'test-bucket' + }); + }); + + it('should handle missing bucket name environment variable', async function () { + delete process.env.dl_s3_bucket_name; + const keyValue = 'test-key'; + const data = { key: 'value' }; + + const params = await getS3ObjectParams(keyValue, data); + + assert.deepStrictEqual(params, { + data: data, + key: keyValue, + bucketName: undefined + }); + }); + }); + + describe('uploadToS3Bucket', function() { + let uploadS3ObjectStub; + let loggerWarnStub; + + beforeEach(function() { + uploadS3ObjectStub = sinon.stub(AlAwsUtil, 'uploadS3Object'); + loggerWarnStub = sinon.stub(AlLogger, 'warn'); + }); + + afterEach(function() { + uploadS3ObjectStub.restore(); + loggerWarnStub.restore(); + }); + + it('should upload data to S3 bucket', function(done) { + const uploadParams = { data: 'test-data', key: 'test-key', bucketName: 'test-bucket' }; + + uploadS3ObjectStub.callsFake((params, callback) => callback(null)); + + uploadToS3Bucket(uploadParams, (err) => { + assert(uploadS3ObjectStub.calledOnce); + assert.strictEqual(err, null); + done(); + }); + }); + + it('should handle upload errors gracefully', function(done) { + const uploadParams = { data: 'test-data', key: 'test-key', bucketName: 'test-bucket' }; + + uploadS3ObjectStub.callsFake((params, callback) => callback(new Error('Upload error'))); + + uploadToS3Bucket(uploadParams, (err) => { + assert(uploadS3ObjectStub.calledOnce); + assert(loggerWarnStub.calledOnce); + assert.strictEqual(err, null); + done(); + }); + }); + + it('should return error if bucket name is missing', function(done) { + const uploadParams = { data: 'test-data', key: 'test-key', bucketName: '' }; + + uploadToS3Bucket(uploadParams, (err) => { + assert.strictEqual(err, 'CMRI0000011 error uploading to undefined bucket'); + done(); + }); + }); + + it('should use default bucket name if not provided', function(done) { + process.env.dl_s3_bucket_name = 'default-bucket'; + const uploadParams = { data: 'test-data', key: 'test-key', bucketName: '' }; + + uploadS3ObjectStub.callsFake((params, callback) => callback(null)); + + uploadToS3Bucket(uploadParams, (err) => { + assert(uploadS3ObjectStub.calledOnce); + assert.strictEqual(err, null); + done(); + }); + }); + }); + +}); diff --git a/collectors/ciscomeraki/utils.js b/collectors/ciscomeraki/utils.js new file mode 100644 index 00000000..a75f8a8e --- /dev/null +++ b/collectors/ciscomeraki/utils.js @@ -0,0 +1,221 @@ +const axios = require('axios'); +const AlLogger = require('@alertlogic/al-aws-collector-js').Logger; + +const { S3Client, GetObjectCommand } = require("@aws-sdk/client-s3"); +const AlAwsUtil = require('@alertlogic/al-aws-collector-js').Util; + +const NETWORKS_PER_PAGE = 1000; +const EVENTS_PER_PAGE = 500; +const API_THROTTLING_ERROR = 429; +const DELAY_IN_SECS = 1000; + +async function getAPILogs(apiDetails, accumulator, apiEndpoint, state, clientSecret, maxPagesPerInvocation) { + let nextPage; + let pageCount = 0; + let since; + return new Promise(async (resolve, reject) => { + try { + for (const productType of apiDetails.productTypes) { + pageCount = 0; + since = state.since; + await getData(productType); + } + return resolve({ accumulator, nextPage }); + } catch (error) { + reject(error); + } + }); + + async function getData(productType) { + if (pageCount < maxPagesPerInvocation) { + pageCount++; + try { + if (state.networkId) { + let url = `https://${apiEndpoint}${apiDetails.url}/${state.networkId}/events`; + let response = await makeApiCall(url, clientSecret, EVENTS_PER_PAGE, productType, since); + let data = response && response.data ? response.data.events : []; + if (data.length) { + accumulator = accumulator.concat(data); + } else { + return accumulator; + } + headers = response.headers; + const linkHeader = response.headers.link; + if (linkHeader && linkHeader.includes('rel=next')) { + const nextLink = linkHeader.match(/<([^>]+)>; rel=next/)[1]; + startingAfter = new URL(nextLink).searchParams.get('startingAfter'); + since = startingAfter; + await getData(productType); + } else { + AlLogger.debug(`CMRI000006 No More Next Page Data Available`); + state.until = response.data.pageEndAt; + } + } + else { + throw new Error(`CMRI000007 Error:NetworkId required in ${url}`); + } + } catch (error) { + throw error; + } + } else { + nextPage = since; + } + } +} + +async function makeApiCall(url, apiKey, perPage, productType, startingAfter = null) { + let fullUrl = `${url}` + + if (perPage) { + fullUrl += `?perPage=${perPage}`; + } + if (productType) { + fullUrl += `&productType=${productType}`; + } + if (startingAfter) { + fullUrl += `&startingAfter=${startingAfter}`; + } + AlLogger.debug(`fullUrl->', ${fullUrl}`); + try { + const response = await axios.get(fullUrl, { + headers: { + "X-Cisco-Meraki-API-Key": apiKey, + "Accept": "application/json" + }, + }); + return response; + } catch (error) { + throw error; + } +} + +async function getAllNetworks(payloadObj) { + const url = `/api/v1/organizations/${payloadObj.orgKey}/networks`; + + try { + const networks = await fetchAllNetworks(url, payloadObj.clientSecret, payloadObj.apiEndpoint); + return networks.map(network => network.id); + } catch (error) { + return error; + } +} + +async function fetchAllNetworks(url, apiKey, apiEndpoint) { + let delay = DELAY_IN_SECS; // Initial delay of 1 second + async function attemptApiCall() { + try { + let response = await makeApiCall(`https://${apiEndpoint}/${url}`, apiKey, NETWORKS_PER_PAGE); + return response.data; + } catch (error) { + if (error.response && error.response.status === API_THROTTLING_ERROR) { + // Rate limit exceeded, applying exponential backoff + await new Promise(resolve => setTimeout(resolve, delay)); + delay *= 2; + return attemptApiCall(); + } else { + throw error; + } + } + } + return attemptApiCall(); +} + +function getOrgKeySecretEndPoint(secret) { + const clientSecret = secret; + if (!clientSecret) { + return "The Client Secret was not found!"; + } + + const apiEndpoint = process.env.paws_endpoint.replace(/^https:\/\/|\/$/g, ''); + const orgKey = process.env.paws_collector_param_string_2; + if (!orgKey) { + return "orgKey was not found!"; + } + + return { clientSecret, apiEndpoint, orgKey }; +} + +function getAPIDetails(orgKey, productTypes) { + let url = "/api/v1/networks"; + let method = "GET"; + let requestBody = ""; + return { + url, + method, + requestBody, + orgKey, + productTypes + }; +} + +function differenceOfNetworksArray(newNetworks, oldNetworks) { + return newNetworks.filter(network => oldNetworks.indexOf(network) === -1); +} + +async function fetchJsonFromS3Bucket(bucketName, fileName, callback) { + try { + const s3Client = new S3Client(); + const getObjectParams = { + Bucket: bucketName, + Key: fileName + }; + const response = await s3Client.send(new GetObjectCommand(getObjectParams)); + + let jsonData = ''; + for await (const chunk of response.Body) { + jsonData += chunk.toString(); + } + const parsedData = JSON.parse(jsonData); + return parsedData; + } catch (error) { + return error; + } +} + +async function getS3ObjectParams(keyValue, data) { + let params = { + data: data, + key: keyValue, + bucketName: process.env.dl_s3_bucket_name + } + return params; +} + +async function uploadNetworksListInS3Bucket(keyValue, networks) { + if (networks.length > 0) { + let params = await getS3ObjectParams(keyValue, networks); + uploadToS3Bucket(params, (err) => { + AlLogger.debug(`CMRI0000010 error while uploading the ${keyValue} : ${JSON.stringify(params)}`); + if (err) { + return err; + } + }); + } +} + +function uploadToS3Bucket({ data, key, bucketName }, callback) { + let bucket = bucketName ? bucketName : process.env.dl_s3_bucket_name; + if (bucket) { + AlAwsUtil.uploadS3Object({ data, key, bucket }, (err) => { + if (err) { + AlLogger.warn(`CMRI0000013 error while uploading the ${key} object in ${bucket} bucket : ${JSON.stringify(err)}`); + } + return callback(null); + }); + } + else return callback(`CMRI0000011 error uploading to ${bucket} bucket`); +} + +module.exports = { + getAPIDetails: getAPIDetails, + makeApiCall: makeApiCall, + getAPILogs: getAPILogs, + getAllNetworks: getAllNetworks, + fetchAllNetworks:fetchAllNetworks, + getOrgKeySecretEndPoint:getOrgKeySecretEndPoint, + uploadNetworksListInS3Bucket: uploadNetworksListInS3Bucket, + getS3ObjectParams: getS3ObjectParams, + uploadToS3Bucket: uploadToS3Bucket, + fetchJsonFromS3Bucket: fetchJsonFromS3Bucket, + differenceOfNetworksArray: differenceOfNetworksArray +}; \ No newline at end of file