🐛 Ensure no duplicate entries

services/export/package.json

@@ -38,6 +38,7 @@
 		"ioredis": "^5.3.2",
 		"json2csv": "^5.0.6",
 		"lisk-service-framework": "https://github.com/LiskHQ/lisk-service/raw/5cb6fc8e9b9798595d1a4652b9148afcbfaaed1f/framework/dist/lisk-service-framework-1.6.11.tgz",
+		"lodash": "^4.17.21",
 		"minio": "^7.0.21",
 		"moment": "^2.29.4",
 		"moment-range": "^4.0.2",

services/export/shared/helpers/array.js

@@ -0,0 +1,22 @@
+/*
+ * LiskHQ/lisk-service
+ * Copyright © 2024 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ *
+ */
+const _ = require('lodash');
+
+const dropDuplicatesDeep = arr => arr.filter((v, i, a) => a.findIndex(t => _.isEqual(v, t)) === i);
+
+module.exports = {
+	dropDuplicatesDeep,
+};

services/export/shared/helpers/transaction.js

@@ -46,14 +46,14 @@ const normalizeTransactionAmount = (address, tx, currentChainID) => {
 	return (sign * amount).toString();
 };
 
-const normalizeTransactionFee = (address, tx) => {
+const normalizeTransactionFee = (addressFromParams, tx) => {
 	const txFee = (BigInt('-1') * BigInt(tx.fee)).toString(); // Fee reduces the balance
 
 	const isTokenTransfer = tx.moduleCommand === `${MODULE.TOKEN}:${COMMAND.TRANSFER}`;
 	if (!isTokenTransfer) return txFee;
 
 	const { isIncomingCrossChainTransferTransaction } = tx;
-	const isRecipient = address === tx.params.recipientAddress;
+	const isRecipient = addressFromParams === tx.params.recipientAddress;
 	return isRecipient || isIncomingCrossChainTransferTransaction ? BigInt('0').toString() : txFee;
 };
 

services/export/shared/transactionsExport.js

@@ -42,6 +42,7 @@ const {
 	getUniqueChainIDs,
 } = require('./helpers');
 
+const { dropDuplicatesDeep } = require('./helpers/array');
 const { checkIfIndexReadyForInterval } = require('./helpers/ready');
 const { standardizeIntervalFromParams } = require('./helpers/time');
 const { requestIndexer, requestAppRegistry } = require('./helpers/request');
@@ -678,7 +679,7 @@ const getEntriesByChronology = async (params, sortedBlocks, sortedTransactions,
 		}
 	}
 
-	return entries;
+	return dropDuplicatesDeep(entries);
 };
 
 const rescheduleExportOnTimeout = async params => {

services/export/tests/unit/shared/helpers/array.test.js

@@ -0,0 +1,55 @@
+/*
+ * LiskHQ/lisk-service
+ * Copyright © 2024 Lisk Foundation
+ *
+ * See the LICENSE file at the top-level directory of this distribution
+ * for licensing information.
+ *
+ * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation,
+ * no part of this software, including this file, may be copied, modified,
+ * propagated, or distributed except according to the terms contained in the
+ * LICENSE file.
+ *
+ * Removal or modification of this copyright notice is prohibited.
+ *
+ */
+const _ = require('lodash');
+const { dropDuplicatesDeep } = require('../../../../shared/helpers/array');
+
+describe('Unit tests for array utilities', () => {
+	describe('Test dropDuplicates method', () => {
+		const isEveryElementUnique = array =>
+			array.every((e, i, a) => a.filter(n => _.isEqual(e, n)).length === 1);
+
+		it('Array with duplicates', async () => {
+			const input1 = [2, 3, 4, 5, 6, 7];
+			const input2 = [5, 7, 9, 11, 13];
+			const result = dropDuplicatesDeep(input1.concat(input2));
+			expect(result).toBeInstanceOf(Array);
+			expect(result.length).toBeLessThanOrEqual(input1.length + input2.length);
+			expect(isEveryElementUnique(result)).toBeTruthy();
+		});
+
+		it('Array with duplicate objects', async () => {
+			const input = [{ a: 1 }, { a: 1 }, { a: 1 }, { b: { c: 2 } }, { b: { c: 2 } }];
+			const result = dropDuplicatesDeep(input);
+			expect(result).toBeInstanceOf(Array);
+			expect(result.length).toBe(2);
+		});
+
+		it('Array with no duplicates', async () => {
+			const input = [2, 3, 4, 5, 6, 7];
+			const result = dropDuplicatesDeep(input);
+			expect(result).toBeInstanceOf(Array);
+			expect(result.length).toBe(input.length);
+			expect(isEveryElementUnique(result)).toBeTruthy();
+		});
+
+		it('Array with no duplicate objects', async () => {
+			const input = [{ a: 1 }, { b: 1 }, { c: 1 }, { b: { c: 2 } }, { b: { c: 200 } }];
+			const result = dropDuplicatesDeep(input);
+			expect(result).toBeInstanceOf(Array);
+			expect(result.length).toBe(5);
+		});
+	});
+});