From 1e74980f0649ffe8998a4bd8996b0d4145a9c0f3 Mon Sep 17 00:00:00 2001 From: Kirill Levitskiy <40201204+kirill-levitskiy@users.noreply.github.com> Date: Fri, 17 Apr 2020 11:34:40 +0300 Subject: [PATCH] New: Write CSV attachment from JSON action (#56) * Add "Write CSV attachment from JSON Array" action * Add "Write CSV attachment from JSON Object" action * Update sailor version to 2.6.5 --- CHANGELOG.md | 6 ++ README.md | 91 +++++++++++++++++++++++- component.json | 116 ++++++++++++++++++++++++++---- lib/actions/writeFromArray.js | 119 +++++++++++++++++++++++++++++++ lib/actions/writeFromJson.js | 129 ++++++++++++++++++++++++++++++++++ package-lock.json | 39 ++++++---- package.json | 10 +-- spec/writeFromArray.spec.js | 79 +++++++++++++++++++++ spec/writeFromJson.spec.js | 79 +++++++++++++++++++++ 9 files changed, 638 insertions(+), 30 deletions(-) create mode 100644 lib/actions/writeFromArray.js create mode 100644 lib/actions/writeFromJson.js create mode 100644 spec/writeFromArray.spec.js create mode 100644 spec/writeFromJson.spec.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 99898f8..cc8d50d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## 2.1.0 (May 7, 2020) + +* Add "Write CSV attachment from Array" action +* Add "Write CSV attachment from JSON" action +* Update sailor version to 2.6.5 + ## 2.0.2 (December 24, 2019) * Update sailor version to 2.5.4 diff --git a/README.md b/README.md index 4c153f2..21bcfa7 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,6 @@ a `JSON` object. To configure this action the following fields can be used: ![image](https://user-images.githubusercontent.com/40201204/60706373-fda1a380-9f11-11e9-8b5a-2acd2df33a87.png) - ### Write CSV attachment * `Include Header` - this select configures output behavior of the component. If option is `Yes` or no value chosen than header of csv file will be written to attachment, this is default behavior. If value `No` selected than csv header will be omitted from attachment. @@ -100,8 +99,98 @@ The output of the CSV Write component will be a message with an attachment. In order to access this attachment, the component following the CSV Write must be able to handle file attachments. +### Write CSV attachment from JSON Object + +* `Include Header` - this select configures output behavior of the component. If option is `Yes` or no value chosen than header of csv file will be written to attachment, this is default behavior. If value `No` selected than csv header will be omitted from attachment. +* `Separator` - this select configures type of CSV delimiter in an output file. There are next options: `Comma (,)`, `Semicolon (;)`, `Space ( )`, `Tab (\t)`. + +This action will combine multiple incoming events into a CSV file until there is a gap +of more than 10 seconds between events. Afterwards, the CSV file will be closed +and attached to the outgoing message. + +This action will convert an incoming array into a CSV file by following approach: + +* Header inherits names of keys from the input message; +* Payload will store data from Values of relevant Keys (Columns); +* Undefined values of a JSON Object won't be joined to result set (`{ key: undefined }`); +* False values of a JSON Object will be represented as empty string (`{ key: false }` => `""`). + +Requirements: + +* The inbound message is an JSON Object; +* This JSON object has plain structure without nested levels (structured types `objects` and `arrays` are not supported as values). Only primitive types are supported: `strings`, `numbers`, `booleans` and `null`. Otherwise, the error message will be thrown: `Inbound message should be a plain Object. At least one of entries is not a primitive type`. + +The keys of an input JSON will be published as the header in the first row. For each incoming +event, the value for each header will be `stringified` and written as the value +for that cell. All other properties will be ignored. For example, headers +`foo,bar` along with the following JSON events: + +``` +{"foo":"myfoo", "bar":"mybar"} +{"foo":"myfoo", "bar":[1,2]} +{"bar":"mybar", "baz":"mybaz"} +``` + +will produce the following `.csv` file: + +``` +foo,bar +myfoo,mybar +myfoo,"[1,2]" +,mybar +``` + +The output of the CSV Write component will be a message with an attachment. In +order to access this attachment, the component following the CSV Write must be +able to handle file attachments. + +### Write CSV attachment from JSON Array + +* `Include Header` - this select configures output behavior of the component. If option is `Yes` or no value chosen than header of csv file will be written to attachment, this is default behavior. If value `No` selected than csv header will be omitted from attachment. +* `Separator` - this select configures type of CSV delimiter in an output file. There are next options: `Comma (,)`, `Semicolon (;)`, `Space ( )`, `Tab (\t)`. + +This action will convert an incoming array into a CSV file by following approach: + +* Header inherits names of keys from the input message; +* Payload will store data from Values of relevant Keys (Columns); +* Undefined values of a JSON Object won't be joined to result set (`{ key: undefined }`); +* False values of a JSON Object will be represented as empty string (`{ key: false }` => `""`). + +Requirements: + +* The inbound message is an JSON Array of Objects with identical structure; +* Each JSON object has plain structure without nested levels (structured types `objects` and `arrays` are not supported as values). Only primitive types are supported: `strings`, `numbers`, `booleans` and `null`. Otherwise, the error message will be thrown: `Inbound message should be a plain Object. At least one of entries is not a primitive type`. + +The keys of an input JSON will be published as the header in the first row. For each incoming +event, the value for each header will be `stringified` and written as the value +for that cell. All other properties will be ignored. For example, headers +`foo,bar` along with the following JSON events: + +``` +[ + {"foo":"myfoo", "bar":"mybar"} + {"foo":"myfoo", "bar":[1,2]} + {"bar":"mybar", "baz":"mybaz"} +] +``` + +will produce the following `.csv` file: + +``` +foo,bar +myfoo,mybar +myfoo2,[1,2]" +,mybar +``` + +The output of the CSV Write component will be a message with an attachment. In +order to access this attachment, the component following the CSV Write must be +able to handle file attachments. + ### Limitations +#### General + 1. You may get `Component run out of memory and terminated.` error during run-time, that means that component needs more memory, please add `EIO_REQUIRED_RAM_MB` environment variable with an appropriate value (e.g. value `512` means that 512 MB will be allocated) for the component in this case. 2. You may get `Error: write after end` error, as a current workaround try increase value of environment variable: `TIMEOUT_BETWEEN_EVENTS`. diff --git a/component.json b/component.json index 7c9b878..98fe636 100644 --- a/component.json +++ b/component.json @@ -2,11 +2,15 @@ "title": "CSV", "description": "A comma-separated values (CSV) file stores tabular data (numbers and text) in plain-text form", "docsUrl": "https://github.com/elasticio/csv-component", - "buildType" : "docker", + "buildType": "docker", "triggers": { "read": { "main": "./lib/triggers/read.js", "title": "Read CSV file from URL", + "help": { + "description": "Fetch a CSV file from a given URL and store it in the attachment storage.", + "link": "/components/csv/index.html#read-csv-file-from-url" + }, "type": "polling", "fields": { "url": { @@ -25,10 +29,14 @@ } } }, - "actions" : { + "actions": { "read_action": { "main": "./lib/triggers/read.js", "title": "Read CSV attachment", + "help": { + "description": "Read a CSV attachment of an incoming message.", + "link": "/components/csv/index.html#read-csv-attachment" + }, "fields": { "emitAll": { "label": "Emit all messages", @@ -44,19 +52,21 @@ } }, "write_attachment": { - "description": - "Multiple incoming events can be combined into one CSV file with the write CSV action. See https://github.com/elasticio/csv-component/ for additional documentation.", "main": "./lib/actions/write.js", "title": "Write CSV attachment", + "help": { + "description": "Multiple incoming events can be combined into one CSV file with the write CSV action.", + "link": "/components/csv/index.html#write-csv-attachment" + }, "fields": { "includeHeaders": { - "label" : "Include Headers", + "label": "Include Headers", "required": false, - "viewClass" : "SelectView", - "description" : "Default Yes", + "viewClass": "SelectView", + "description": "Default Yes", "model": { - "Yes" : "Yes", - "No" : "No" + "Yes": "Yes", + "No": "No" }, "prompt": "Include headers? Default Yes." }, @@ -66,11 +76,93 @@ }, "metadata": { "in": { - "type": "object", - "properties": {} + "type": "object", + "properties": {} + }, + "out": {} + } + }, + "write_attachment_from_json": { + "main": "./lib/actions/writeFromJson.js", + "title": "Write CSV attachment from JSON Object", + "help": { + "description": "Multiple incoming events can be combined into one CSV file with the write CSV action.", + "link": "/components/csv/index.html#write-csv-attachment-from-json" + }, + "fields": { + "includeHeaders": { + "label": "Include Headers", + "required": true, + "viewClass": "SelectView", + "description": "Default Yes", + "model": { + "Yes": "Yes", + "No": "No" + }, + "prompt": "Include headers? Default Yes" + }, + "separator": { + "label": "Separators", + "required": true, + "viewClass": "SelectView", + "description": "Default Yes", + "model": { + "comma": "Comma (,)", + "semicolon": "Semicolon (;)", + "space": "Space ( )", + "tab": "Tab (\\t)" + }, + "prompt": "Choose required CSV delimiter" + } + }, + "metadata": { + "in": { + "type": "object", + "properties": {} + }, + "out": {} + } + }, + "write_attachment_from_array": { + "main": "./lib/actions/writeFromArray.js", + "title": "Write CSV attachment from JSON Array", + "help": { + "description": "Incoming array can be converted into one CSV file with the write CSV action.", + "link": "/components/csv/index.html#write-csv-attachment-from-array" + }, + "fields": { + "includeHeaders": { + "label": "Include Headers", + "required": true, + "viewClass": "SelectView", + "description": "Default Yes", + "model": { + "Yes": "Yes", + "No": "No" + }, + "prompt": "Include headers? Default Yes" + }, + "separator": { + "label": "Separators", + "required": true, + "viewClass": "SelectView", + "description": "Default Yes", + "model": { + "comma": "Comma (,)", + "semicolon": "Semicolon (;)", + "space": "Space ( )", + "tab": "Tab (\\t)" + }, + "prompt": "Choose required CSV delimiter" + } + }, + "metadata": { + "in": { + "type": "array", + "properties": {} }, "out": {} } } } -} +} \ No newline at end of file diff --git a/lib/actions/writeFromArray.js b/lib/actions/writeFromArray.js new file mode 100644 index 0000000..e124be8 --- /dev/null +++ b/lib/actions/writeFromArray.js @@ -0,0 +1,119 @@ +const axios = require('axios'); +const csv = require('csv'); +const _ = require('lodash'); +const { messages } = require('elasticio-node'); +const client = require('elasticio-rest-node')(); +const logger = require('@elastic.io/component-logger')(); + +const util = require('../util/util'); + +const REQUEST_TIMEOUT = process.env.REQUEST_TIMEOUT || 10000; // 10s +const REQUEST_MAX_RETRY = process.env.REQUEST_MAX_RETRY || 7; +const REQUEST_RETRY_DELAY = process.env.REQUEST_RETRY_DELAY || 7000; // 7s +const REQUEST_MAX_CONTENT_LENGTH = process.env.REQUEST_MAX_CONTENT_LENGTH || 10485760; // 10MB + +let stringifier; +let signedUrl; +let rowCount = 0; +let ax; +let putUrl; +let options; + +async function init(cfg) { + let delimiter; + switch (cfg.separator) { + case 'comma': { + delimiter = ','; + break; + } + case 'semicolon': { + delimiter = ';'; + break; + } + case 'space': { + delimiter = ' '; + break; + } + case 'tab': { + delimiter = '\t'; + break; + } + default: { + throw Error(`Unexpected separator type: ${cfg.separator}`); + } + } + const header = cfg.includeHeaders !== 'No'; + logger.trace('Using delimiter: \'%s\'', delimiter); + options = { + header, + delimiter, + }; + + stringifier = csv.stringify(options); + signedUrl = await client.resources.storage.createSignedUrl(); + putUrl = signedUrl.put_url; + logger.trace('CSV file to be uploaded file to uri=%s', putUrl); + ax = axios.create(); + util.addRetryCountInterceptorToAxios(ax); +} +async function ProcessAction(msg) { + // eslint-disable-next-line consistent-this + const self = this; + let isError = false; + let errorValue = ''; + + const columns = Object.keys(msg.body[0]); + rowCount = msg.body.length; + logger.trace('Configured column names:', columns); + let row = {}; + + await _.each(msg.body, async (item) => { + const entries = Object.values(msg.body); + // eslint-disable-next-line no-restricted-syntax + for (const entry of entries) { + if (isError) { + break; + } + const values = Object.values(entry); + // eslint-disable-next-line no-restricted-syntax + for (const value of values) { + if (value !== null && value !== undefined && (typeof value === 'object' || Array.isArray(value))) { + isError = true; + errorValue = value; + break; + } + } + } + row = _.pick(item, columns); + await stringifier.write(row); + }); + self.logger.info('The resulting CSV file contains %s rows', rowCount); + + if (isError) { + throw Error(`Inbound message should be a plain Object. At least one of entries is not a primitive type: ${JSON.stringify(errorValue)}`); + } + + ax.put(putUrl, stringifier, { + method: 'PUT', + timeout: REQUEST_TIMEOUT, + retry: REQUEST_MAX_RETRY, + delay: REQUEST_RETRY_DELAY, + maxContentLength: REQUEST_MAX_CONTENT_LENGTH, + }); + stringifier.end(); + + const messageToEmit = messages.newMessageWithBody({ + rowCount, + }); + const fileName = `${messageToEmit.id}.csv`; + messageToEmit.attachments[fileName] = { + 'content-type': 'text/csv', + url: signedUrl.get_url, + }; + self.logger.trace('Emitting message %j', messageToEmit); + await self.emit('data', messageToEmit); + await self.emit('end'); +} + +exports.process = ProcessAction; +exports.init = init; diff --git a/lib/actions/writeFromJson.js b/lib/actions/writeFromJson.js new file mode 100644 index 0000000..01caeb2 --- /dev/null +++ b/lib/actions/writeFromJson.js @@ -0,0 +1,129 @@ +const axios = require('axios'); +const csv = require('csv'); +const _ = require('lodash'); +const { messages } = require('elasticio-node'); +const client = require('elasticio-rest-node')(); +const logger = require('@elastic.io/component-logger')(); + +const util = require('../util/util'); + +const REQUEST_TIMEOUT = process.env.REQUEST_TIMEOUT || 10000; // 10s +const REQUEST_MAX_RETRY = process.env.REQUEST_MAX_RETRY || 7; +const REQUEST_RETRY_DELAY = process.env.REQUEST_RETRY_DELAY || 7000; // 7s +const REQUEST_MAX_CONTENT_LENGTH = process.env.REQUEST_MAX_CONTENT_LENGTH || 10485760; // 10MB +const TIMEOUT_BETWEEN_EVENTS = process.env.TIMEOUT_BETWEEN_EVENTS || 10000; // 10s; + +let stringifier; +let signedUrl; +let timeout; +let rowCount = 0; +let ax; +let putUrl; +let options; + +let readyFlag = false; + +async function init(cfg) { + let delimiter; + switch (cfg.separator) { + case 'comma': { + delimiter = ','; + break; + } + case 'semicolon': { + delimiter = ';'; + break; + } + case 'space': { + delimiter = ' '; + break; + } + case 'tab': { + delimiter = '\t'; + break; + } + default: { + throw Error(`Unexpected separator type: ${cfg.separator}`); + } + } + const header = cfg.includeHeaders !== 'No'; + logger.trace('Using delimiter: \'%s\'', delimiter); + options = { + header, + delimiter, + }; + + stringifier = csv.stringify(options); + signedUrl = await client.resources.storage.createSignedUrl(); + putUrl = signedUrl.put_url; + logger.trace('CSV file to be uploaded file to uri=%s', putUrl); + ax = axios.create(); + util.addRetryCountInterceptorToAxios(ax); + readyFlag = true; +} +async function ProcessAction(msg) { + // eslint-disable-next-line consistent-this + const self = this; + + const columns = Object.keys(msg.body); + logger.trace('Configured column names:', columns); + + const values = Object.values(msg.body); + // eslint-disable-next-line no-restricted-syntax + for (const value of values) { + if (value !== null && value !== undefined && (typeof value === 'object' || Array.isArray(value))) { + throw Error(`Inbound message should be a plain Object. At least one of entries is not a primitive type:\n${JSON.stringify(value)}`); + } + } + + while (!readyFlag) { + // eslint-disable-next-line no-loop-func,no-await-in-loop + await new Promise(resolve => timeout(resolve, 100)); + } + + if (timeout) { + clearTimeout(timeout); + } + + timeout = setTimeout(async () => { + readyFlag = false; + self.logger.info('Closing the stream due to inactivity'); + + const finalRowCount = rowCount; + self.logger.info('The resulting CSV file contains %s rows', finalRowCount); + ax.put(putUrl, stringifier, { + method: 'PUT', + timeout: REQUEST_TIMEOUT, + retry: REQUEST_MAX_RETRY, + delay: REQUEST_RETRY_DELAY, + maxContentLength: REQUEST_MAX_CONTENT_LENGTH, + }); + stringifier.end(); + + const messageToEmit = messages.newMessageWithBody({ + rowCount: finalRowCount, + }); + const fileName = `${messageToEmit.id}.csv`; + messageToEmit.attachments[fileName] = { + 'content-type': 'text/csv', + url: signedUrl.get_url, + }; + signedUrl = null; + rowCount = 0; + self.logger.trace('Emitting message %j', messageToEmit); + await self.emit('data', messageToEmit); + }, TIMEOUT_BETWEEN_EVENTS); + + let row = msg.body; + self.logger.trace(`Incoming data: ${JSON.stringify(row)}`); + row = _.pick(row, columns); + + self.logger.trace(`Writing Row: ${JSON.stringify(row)}`); + stringifier.write(row); + rowCount += 1; + + await self.emit('end'); +} + +exports.process = ProcessAction; +exports.init = init; diff --git a/package-lock.json b/package-lock.json index 4e6f1ed..d672f98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "csv-component", - "version": "2.0.2", + "version": "2.1.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -51,6 +51,7 @@ "version": "1.6.0", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.6.0.tgz", "integrity": "sha512-w4/WHG7C4WWFyE5geCieFJF6MZkbW4VAriol5KlmQXpAQdxvV0p26sqNZOW6Qyw6Y0l9K4g+cHvvczR2sEEpqg==", + "dev": true, "requires": { "type-detect": "4.0.8" } @@ -59,6 +60,7 @@ "version": "3.2.1", "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-3.2.1.tgz", "integrity": "sha512-tsHvOB24rvyvV2+zKMmPkZ7dXX6LSLKZ7aOtXY6Edklp0uRcgGpOsQTTGTcWViFyx4uhWc6GV8QdnALbIbIdeQ==", + "dev": true, "requires": { "@sinonjs/commons": "^1", "@sinonjs/samsam": "^3.1.0" @@ -68,6 +70,7 @@ "version": "3.3.3", "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-3.3.3.tgz", "integrity": "sha512-bKCMKZvWIjYD0BLGnNrxVuw4dkWCYsLqFOUWw8VgKF/+5Y+mE7LfHWPIYoDXowH+3a9LsWDMo0uAP8YDosPvHQ==", + "dev": true, "requires": { "@sinonjs/commons": "^1.3.0", "array-from": "^2.1.1", @@ -77,14 +80,16 @@ "lodash": { "version": "4.17.15", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz", - "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==" + "integrity": "sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A==", + "dev": true } } }, "@sinonjs/text-encoding": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", - "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==" + "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", + "dev": true }, "accounting": { "version": "0.4.1", @@ -174,7 +179,8 @@ "array-from": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/array-from/-/array-from-2.1.1.tgz", - "integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=" + "integrity": "sha1-z+nYwmYoudxa7MYqn12PHzUsEZU=", + "dev": true }, "array-includes": { "version": "3.0.3", @@ -707,9 +713,9 @@ } }, "elasticio-sailor-nodejs": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/elasticio-sailor-nodejs/-/elasticio-sailor-nodejs-2.5.4.tgz", - "integrity": "sha512-8ne1d/qfAcvI12nG4X2yY4rGfyCYkf4Q0xCtAsO+W0sXYHOy9HPIIi2sGesFxm67HrDCPtPIwrsecwOl61L8xA==", + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/elasticio-sailor-nodejs/-/elasticio-sailor-nodejs-2.6.5.tgz", + "integrity": "sha512-TNSCHPgDNfCDFqQ7kfA9fB5ERMajj6D7Ecql3azsCle07R1BJMzXzcH3nWrq4Jx/fGk+1vNK7PxDd9sAXokWQQ==", "requires": { "amqplib": "0.5.1", "bunyan": "1.8.10", @@ -1301,7 +1307,8 @@ "has-flag": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true }, "has-symbols": { "version": "1.0.0", @@ -1559,7 +1566,8 @@ "just-extend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", - "integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==" + "integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==", + "dev": true }, "lcid": { "version": "2.0.0", @@ -1619,7 +1627,8 @@ "lolex": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/lolex/-/lolex-4.2.0.tgz", - "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==" + "integrity": "sha512-gKO5uExCXvSm6zbF562EvM+rd1kQDnB9AZBbiQVzf1ZmdDpxUSvpnAaVOP83N/31mRK8Ml8/VE8DMvsAZQ+7wg==", + "dev": true }, "map-age-cleaner": { "version": "0.1.3", @@ -1891,6 +1900,7 @@ "version": "1.5.2", "resolved": "https://registry.npmjs.org/nise/-/nise-1.5.2.tgz", "integrity": "sha512-/6RhOUlicRCbE9s+94qCUsyE+pKlVJ5AhIv+jEE7ESKwnbXqulKZ1FYU+XAtHHWE9TinYvAxDUJAb912PwPoWA==", + "dev": true, "requires": { "@sinonjs/formatio": "^3.2.1", "@sinonjs/text-encoding": "^0.7.1", @@ -2205,6 +2215,7 @@ "version": "1.7.0", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.7.0.tgz", "integrity": "sha1-Wf3g9DW62suhA6hOnTvGTpa5k30=", + "dev": true, "requires": { "isarray": "0.0.1" } @@ -2551,6 +2562,7 @@ "version": "7.5.0", "resolved": "https://registry.npmjs.org/sinon/-/sinon-7.5.0.tgz", "integrity": "sha512-AoD0oJWerp0/rY9czP/D6hDTTUYGpObhZjMpd7Cl/A6+j0xBE+ayL/ldfggkBXUs0IkvIiM1ljM8+WkOc5k78Q==", + "dev": true, "requires": { "@sinonjs/commons": "^1.4.0", "@sinonjs/formatio": "^3.2.1", @@ -2564,7 +2576,8 @@ "diff": { "version": "3.5.0", "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", - "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==" + "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", + "dev": true } } }, @@ -2729,6 +2742,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, "requires": { "has-flag": "^3.0.0" } @@ -2890,7 +2904,8 @@ "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==" + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true }, "underscore": { "version": "1.8.3", diff --git a/package.json b/package.json index 16b4ba1..1bd2b3c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "csv-component", - "version": "2.0.2", + "version": "2.1.0", "description": "CSV Component for elastic.io platform", "main": "index.js", "scripts": { @@ -9,7 +9,7 @@ "integration-test": "mocha spec-integration" }, "engines": { - "node": "8.1.0" + "node": "12" }, "repository": { "type": "git", @@ -33,12 +33,11 @@ "csv-parser": "2.2.0", "elasticio-node": "0.0.9", "elasticio-rest-node": "1.2.3", - "elasticio-sailor-nodejs": "2.5.4", + "elasticio-sailor-nodejs": "2.6.5", "lodash": "4.17.13", "moment": "2.24.0", "node-uuid": "1.4.3", "q": "1.4.1", - "sinon": "7.5.0", "stream": "0.0.2", "@elastic.io/component-logger": "0.0.1", "underscore": "1.8.3" @@ -52,6 +51,7 @@ "eslint-plugin-import": "^2.17.3", "eslint-plugin-mocha": "6.0.0", "mocha": "6.1.4", - "nock": "9.0.13" + "nock": "9.0.13", + "sinon": "7.5.0" } } diff --git a/spec/writeFromArray.spec.js b/spec/writeFromArray.spec.js new file mode 100644 index 0000000..031bf4e --- /dev/null +++ b/spec/writeFromArray.spec.js @@ -0,0 +1,79 @@ +/* eslint-disable no-unused-vars */ +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +const fs = require('fs'); +const nock = require('nock'); +const sinon = require('sinon'); +const logger = require('@elastic.io/component-logger')(); + +chai.use(chaiAsPromised); +const { expect } = require('chai'); + +if (fs.existsSync('.env')) { + // eslint-disable-next-line global-require + require('dotenv').config(); +} else { + process.env.ELASTICIO_API_USERNAME = 'name'; + process.env.ELASTICIO_API_KEY = 'key'; +} + +const write = require('../lib/actions/writeFromArray.js'); + +describe('CSV Write From Array component', function () { + this.timeout(15000); + + let emit; + let cfg; + + before(async () => { + cfg = { + includeHeaders: 'Yes', + separator: 'semicolon', + }; + + nock('https://api.elastic.io', { encodedQueryParams: true }) + .post('/v2/resources/storage/signed-url') + .reply(200, + { put_url: 'https://examlple.mock/putUrl', get_url: 'https://examlple.mock/getUrl' }); + + nock('https://examlple.mock') + .put('/putUrl') + .reply(200, {}); + }); + + beforeEach(() => { + emit = sinon.spy(); + }); + + it('should write csv rows', async () => { + await write.init.call({ + logger, + }, cfg); + + const msg = { + body: [ + { + name: 'Bob', + email: 'bob@email.domain', + age: 30, + key1: true, + 'not an age': null, + 'not an age at all': undefined, + }, + { + name: 'Joe', + email: 'joe@email.domain', + age: 11, + 'not an age at all': 322, + }, + ], + }; + + await write.process.call({ + emit, + logger, + }, msg, cfg); + + expect(emit.getCalls().length).to.equal(2); + }); +}); diff --git a/spec/writeFromJson.spec.js b/spec/writeFromJson.spec.js new file mode 100644 index 0000000..33c12f6 --- /dev/null +++ b/spec/writeFromJson.spec.js @@ -0,0 +1,79 @@ +/* eslint-disable no-unused-vars */ +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +const fs = require('fs'); +const nock = require('nock'); +const sinon = require('sinon'); +const logger = require('@elastic.io/component-logger')(); + +chai.use(chaiAsPromised); +const { expect } = require('chai'); + +if (fs.existsSync('.env')) { + // eslint-disable-next-line global-require + require('dotenv').config(); +} else { + process.env.ELASTICIO_API_USERNAME = 'name'; + process.env.ELASTICIO_API_KEY = 'key'; +} + +const write = require('../lib/actions/writeFromJson.js'); + +describe('CSV Write From JSON component', function () { + this.timeout(15000); + + let emit; + let cfg; + + before(async () => { + cfg = { + includeHeaders: 'Yes', + separator: 'semicolon', + }; + + nock('https://api.elastic.io', { encodedQueryParams: true }) + .post('/v2/resources/storage/signed-url') + .reply(200, + { put_url: 'https://examlple.mock/putUrl', get_url: 'https://examlple.mock/getUrl' }); + + nock('https://examlple.mock') + .put('/putUrl') + .reply(200, {}); + }); + + beforeEach(() => { + emit = sinon.spy(); + }); + + it('should write csv rows', async () => { + await write.init.call({ + logger, + }, cfg); + + const msg1 = { + body: { + ProductKey: 'text11', + CategoryGroup_1: 'text12', + }, + }; + + await write.process.call({ + emit, + logger, + }, msg1, cfg); + + const msg2 = { + body: { + ProductKey: 'text21', + CategoryGroup_1: 'text22', + }, + }; + + await write.process.call({ emit, logger }, msg2, cfg); + + expect(emit.getCalls().length).to.equal(2); + + await new Promise(resolve => setTimeout(resolve, 12000)); + expect(emit.getCalls().length).to.equal(3); + }); +});