diff --git a/services/blockchain-connector/methods/token.js b/services/blockchain-connector/methods/token.js index ca5991337a..d5efec1931 100644 --- a/services/blockchain-connector/methods/token.js +++ b/services/blockchain-connector/methods/token.js @@ -22,6 +22,7 @@ const { getTokenBalances, getTokenInitializationFees, tokenHasEscrowAccount, + getTokenBalanceAtGenesis, } = require('../shared/sdk'); module.exports = [ @@ -84,4 +85,11 @@ module.exports = [ controller: async () => getTokenInitializationFees(), params: {}, }, + { + name: 'getTokenBalanceAtGenesis', + controller: async ({ address }) => getTokenBalanceAtGenesis(address), + params: { + address: { optional: false, type: 'string' }, + }, + }, ]; diff --git a/services/blockchain-connector/shared/sdk/index.js b/services/blockchain-connector/shared/sdk/index.js index 76c18b272c..47ca1c5d80 100644 --- a/services/blockchain-connector/shared/sdk/index.js +++ b/services/blockchain-connector/shared/sdk/index.js @@ -65,6 +65,7 @@ const { getTotalSupply, getTokenInitializationFees, updateTokenInfo, + getTokenBalanceAtGenesis, } = require('./token'); const { @@ -178,7 +179,7 @@ module.exports = { dryRunTransaction, formatTransaction, - // Tokens + // Token tokenHasUserAccount, tokenHasEscrowAccount, getTokenBalance, @@ -187,6 +188,7 @@ module.exports = { getSupportedTokens, getTotalSupply, getTokenInitializationFees, + getTokenBalanceAtGenesis, // PoS getAllPosValidators, diff --git a/services/blockchain-connector/shared/sdk/token.js b/services/blockchain-connector/shared/sdk/token.js index 11468ad978..5db1ddbbcd 100644 --- a/services/blockchain-connector/shared/sdk/token.js +++ b/services/blockchain-connector/shared/sdk/token.js @@ -15,6 +15,8 @@ */ const { invokeEndpoint } = require('./client'); const { isMainchain } = require('./interoperability'); +const { getGenesisAssetByModule } = require('./genesisBlock'); +const { MODULE_NAME_TOKEN } = require('./constants/names'); let escrowedAmounts; let supportedTokens; @@ -73,6 +75,20 @@ const updateTokenInfo = async () => { totalSupply = await getTotalSupply(true); }; +const getTokenBalanceAtGenesis = async address => { + const MODULE_TOKEN_SUBSTORE_USER = 'userSubstore'; + + const tokenModuleGenesisAssets = await getGenesisAssetByModule({ + module: MODULE_NAME_TOKEN, + subStore: MODULE_TOKEN_SUBSTORE_USER, + }); + + const balancesAtGenesis = tokenModuleGenesisAssets[MODULE_TOKEN_SUBSTORE_USER]; + const balancesByAddress = balancesAtGenesis.find(e => e.address === address); + + return balancesByAddress; +}; + module.exports = { tokenHasUserAccount: hasUserAccount, tokenHasEscrowAccount: hasEscrowAccount, @@ -83,4 +99,5 @@ module.exports = { getTotalSupply, getTokenInitializationFees, updateTokenInfo, + getTokenBalanceAtGenesis, }; diff --git a/services/blockchain-indexer/methods/dataService/modules/token.js b/services/blockchain-indexer/methods/dataService/modules/token.js index ae31133a8a..a778e29539 100644 --- a/services/blockchain-indexer/methods/dataService/modules/token.js +++ b/services/blockchain-indexer/methods/dataService/modules/token.js @@ -61,7 +61,8 @@ module.exports = [ address: { optional: true, type: 'string', pattern: regex.ADDRESS_LISK32 }, publicKey: { optional: true, type: 'string', pattern: regex.PUBLIC_KEY }, name: { optional: true, type: 'string', pattern: regex.NAME }, - tokenID: { optional: false, type: 'string', pattern: regex.TOKEN_ID }, + // Set tokenID as optional in indexer because export microservice needs it to be optional. Should remain mandatory everywhere else. + tokenID: { optional: true, type: 'string', pattern: regex.TOKEN_ID }, }, }, { diff --git a/services/blockchain-indexer/shared/dataService/business/token/accountExists.js b/services/blockchain-indexer/shared/dataService/business/token/accountExists.js index be8dc11b26..8e51131020 100644 --- a/services/blockchain-indexer/shared/dataService/business/token/accountExists.js +++ b/services/blockchain-indexer/shared/dataService/business/token/accountExists.js @@ -17,6 +17,7 @@ const { requestConnector } = require('../../../utils/request'); const { getAddressByName } = require('../../utils/validator'); const { getLisk32AddressFromPublicKey } = require('../../../utils/account'); +const { getAvailableTokenIDs } = require('./availableIDs'); const tokenHasUserAccount = async params => { const response = { @@ -26,7 +27,6 @@ const tokenHasUserAccount = async params => { meta: {}, }; - const { tokenID } = params; let { address } = params; if (!address && params.name) { @@ -39,11 +39,28 @@ const tokenHasUserAccount = async params => { // Check existence if address found. Return false otherwise if (address) { - const { exists: isExists } = await requestConnector('tokenHasUserAccount', { - address, - tokenID, - }); - response.data.isExists = isExists; + const tokenIDs = []; + + if (params.tokenID) { + tokenIDs.push(params.tokenID); + } else { + // Logic introduced to support the export microservice + const result = await getAvailableTokenIDs(); + tokenIDs.push(...result.data.tokenIDs); + } + + // eslint-disable-next-line no-restricted-syntax + for (const tokenID of tokenIDs) { + const { exists: isExists } = await requestConnector('tokenHasUserAccount', { + address, + tokenID, + }); + + if (isExists) { + response.data.isExists = isExists; + break; + } + } } return response; diff --git a/services/blockchain-indexer/shared/dataService/business/token/availableIDs.js b/services/blockchain-indexer/shared/dataService/business/token/availableIDs.js index 87faed5eb1..6aa5e8d77c 100644 --- a/services/blockchain-indexer/shared/dataService/business/token/availableIDs.js +++ b/services/blockchain-indexer/shared/dataService/business/token/availableIDs.js @@ -26,7 +26,7 @@ const MYSQL_ENDPOINT = config.endpoints.mysqlReplica; const getAccountBalancesTable = () => getTableInstance(accountBalancesTableSchema, MYSQL_ENDPOINT); -const getAvailableTokenIDs = async params => { +const getAvailableTokenIDs = async (params = {}) => { const response = { data: {}, meta: {}, diff --git a/services/blockchain-indexer/shared/indexer/genesisBlock.js b/services/blockchain-indexer/shared/indexer/genesisBlock.js index 8dfe8ff79c..717c79fe12 100644 --- a/services/blockchain-indexer/shared/indexer/genesisBlock.js +++ b/services/blockchain-indexer/shared/indexer/genesisBlock.js @@ -197,6 +197,7 @@ const indexPosModuleAssets = async dbTrx => { }; const indexGenesisBlockAssets = async dbTrx => { + clearTimeout(intervalTimeout); logger.info('Starting to index the genesis assets.'); intervalTimeout = setInterval( () => logger.info('Genesis assets indexing still in progress...'), diff --git a/services/blockchain-indexer/shared/utils/transactions.js b/services/blockchain-indexer/shared/utils/transactions.js index 7355bbae25..f6ed0e4085 100644 --- a/services/blockchain-indexer/shared/utils/transactions.js +++ b/services/blockchain-indexer/shared/utils/transactions.js @@ -28,7 +28,9 @@ const getTransactionExecutionStatus = (tx, events) => { e => e.topics.includes(EVENT_TOPIC_PREFIX.TX_ID.concat(tx.id)) || e.topics.includes(tx.id), ); if (!txExecResultEvent) - throw Error(`Event unavailable to determine execution status for transaction: ${tx.id}.`); + throw Error( + `Event unavailable to determine execution status for transaction: ${tx.id}.\nEnsure that you have set 'system.keepEventsForHeights: -1' in your node config before syncing it with the network.`, + ); return txExecResultEvent.data.success ? TRANSACTION_STATUS.SUCCESSFUL : TRANSACTION_STATUS.FAILED; }; diff --git a/services/blockchain-indexer/tests/unit/shared/dataservice/business/token/accountExists.test.js b/services/blockchain-indexer/tests/unit/shared/dataservice/business/token/accountExists.test.js new file mode 100644 index 0000000000..38b5f97b61 --- /dev/null +++ b/services/blockchain-indexer/tests/unit/shared/dataservice/business/token/accountExists.test.js @@ -0,0 +1,156 @@ +/* + * LiskHQ/lisk-service + * Copyright © 2024 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + * + */ + +/* eslint-disable import/no-dynamic-require */ +const { resolve } = require('path'); + +const mockRequestFilePath = resolve(`${__dirname}/../../../../../../shared/utils/request`); +const mockTokenAvailableIDsFilePath = resolve( + `${__dirname}/../../../../../../shared/dataService/business/token/availableIDs`, +); +const mockTokenaccountExistsFilePath = resolve( + `${__dirname}/../../../../../../shared/dataService/business/token/accountExists`, +); +const mockValidatorUtilsPath = resolve( + `${__dirname}/../../../../../../shared/dataService/utils/validator`, +); + +beforeEach(() => jest.resetModules()); + +jest.mock('lisk-service-framework', () => { + const actual = jest.requireActual('lisk-service-framework'); + return { + ...actual, + DB: { + ...actual.DB, + MySQL: { + ...actual.DB.MySQL, + KVStore: { + ...actual.DB.MySQL.KVStore, + getKeyValueTable: jest.fn(), + }, + }, + }, + CacheRedis: jest.fn(), + CacheLRU: jest.fn(), + }; +}); + +describe('tokenHasUserAccount', () => { + const tokenID = '0000000000000000'; + const accAddressExists = 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo'; + const accAddressNotExists = 'lskz23xokaxhmmkpbzdjt5agcq59qkby7bne2hwpk'; + const name = 'testAccount'; + const publicKey = '3972849f2ab66376a68671c10a00e8b8b67d880434cc65b04c6ed886dfa91c2c'; + + describe('should return isExists true when user account exists', () => { + it('when called with tokenID and address', async () => { + jest.mock(mockRequestFilePath, () => ({ + requestConnector: jest.fn(() => ({ exists: true })), + })); + + // Make a query to tokenHasUserAccount function + const { tokenHasUserAccount } = require(mockTokenaccountExistsFilePath); + const result = await tokenHasUserAccount({ address: accAddressExists, tokenID }); + + expect(result).toEqual({ + data: { + isExists: true, + }, + meta: {}, + }); + }); + + it('when called with tokenID and publicKey', async () => { + jest.mock(mockRequestFilePath, () => ({ + requestConnector: jest.fn(() => ({ exists: true })), + })); + + // Make a query to tokenHasUserAccount function + const { tokenHasUserAccount } = require(mockTokenaccountExistsFilePath); + const result = await tokenHasUserAccount({ publicKey, tokenID }); + expect(result).toEqual({ + data: { + isExists: true, + }, + meta: {}, + }); + }); + + it('when called with tokenID and name', async () => { + jest.mock(mockRequestFilePath, () => ({ + requestConnector: jest.fn(() => ({ exists: true })), + })); + + jest.mock(mockValidatorUtilsPath); + const { getAddressByName } = require(mockValidatorUtilsPath); + getAddressByName.mockReturnValueOnce(accAddressExists); + + // Make a query to tokenHasUserAccount function + const { tokenHasUserAccount } = require(mockTokenaccountExistsFilePath); + const result = await tokenHasUserAccount({ name, tokenID }); + expect(result).toEqual({ + data: { + isExists: true, + }, + meta: {}, + }); + }); + + it('when called with address', async () => { + jest.mock(mockRequestFilePath, () => ({ + requestConnector: jest.fn(() => ({ exists: true })), + })); + + jest.mock(mockTokenAvailableIDsFilePath); + const { getAvailableTokenIDs } = require(mockTokenAvailableIDsFilePath); + getAvailableTokenIDs.mockReturnValueOnce({ + data: { tokenIDs: ['0000000000000000'] }, + meta: {}, + }); + + // Make a query to tokenHasUserAccount function + const { tokenHasUserAccount } = require(mockTokenaccountExistsFilePath); + const result = await tokenHasUserAccount({ address: accAddressExists }); + + expect(result).toEqual({ + data: { + isExists: true, + }, + meta: {}, + }); + }); + }); + + describe('should return isExists false when user account does not exists', () => { + it('when called with tokenID and address', async () => { + jest.mock(mockRequestFilePath, () => ({ + requestConnector: jest.fn(() => ({ exists: false })), + })); + + // Make a query to tokenHasUserAccount function + const { tokenHasUserAccount } = require(mockTokenaccountExistsFilePath); + const result = await tokenHasUserAccount({ address: accAddressNotExists, tokenID }); + + expect(result).toEqual({ + data: { + isExists: false, + }, + meta: {}, + }); + }); + }); +}); diff --git a/services/export/app.js b/services/export/app.js index 4b046972c2..28c5122dea 100644 --- a/services/export/app.js +++ b/services/export/app.js @@ -14,7 +14,7 @@ * */ const path = require('path'); -const { Microservice, LoggerConfig, Logger } = require('lisk-service-framework'); +const { Signals, Microservice, LoggerConfig, Logger } = require('lisk-service-framework'); const config = require('./config'); @@ -22,6 +22,7 @@ LoggerConfig(config.log); const packageJson = require('./package.json'); const { setAppContext } = require('./shared/helpers'); +const { getTokenBalancesAtGenesis } = require('./shared/transactionsExport'); const logger = Logger(); @@ -32,6 +33,17 @@ const app = Microservice({ timeout: config.brokerTimeout, packageJson, logger: config.log, + events: { + systemNodeInfo: async payload => { + logger.debug("Received a 'systemNodeInfo' moleculer event from connecter."); + Signals.get('nodeInfo').dispatch(payload); + }, + 'update.index.status': async payload => { + logger.debug("Received a 'update.index.status' moleculer event from indexer."); + Signals.get('updateIndexStatus').dispatch(payload); + }, + }, + dependencies: ['connector', 'indexer', 'app-registry'], }); setAppContext(app); @@ -43,8 +55,9 @@ app.addJobs(path.join(__dirname, 'jobs')); // Run the application app .run() - .then(() => { + .then(async () => { logger.info(`Service started ${packageJson.name}.`); + await getTokenBalancesAtGenesis(); }) .catch(err => { logger.fatal(`Failed to start service ${packageJson.name} due to: ${err.message}`); diff --git a/services/export/config.js b/services/export/config.js index d81b58cce4..51ac232980 100644 --- a/services/export/config.js +++ b/services/export/config.js @@ -78,7 +78,20 @@ config.excel.sheets = { config.queue = { scheduleTransactionExport: { name: 'ScheduleTransactionExportQueue', - concurrency: 50, + concurrency: 10, + options: { + defaultJobOptions: { + attempts: 5, + timeout: 5 * 60 * 1000, // millisecs + backoff: { + type: 'exponential', + delay: 1 * 60 * 1000, // millisecs + }, + removeOnComplete: true, + removeOnFail: true, + stackTraceLimit: 0, + }, + }, }, defaults: { jobOptions: { diff --git a/services/export/shared/helpers/chain.js b/services/export/shared/helpers/chain.js index 901e6e4138..9fa5629d81 100644 --- a/services/export/shared/helpers/chain.js +++ b/services/export/shared/helpers/chain.js @@ -45,9 +45,15 @@ const getUniqueChainIDs = async txs => { return Array.from(chainIDs); }; +const getBlocks = async params => requestIndexer('blocks', params); + +const getTransactions = async params => requestIndexer('transactions', params); + module.exports = { getCurrentChainID, resolveReceivingChainID, getNetworkStatus, getUniqueChainIDs, + getBlocks, + getTransactions, }; diff --git a/services/export/shared/helpers/constants.js b/services/export/shared/helpers/constants.js index 59555af7be..048720297a 100644 --- a/services/export/shared/helpers/constants.js +++ b/services/export/shared/helpers/constants.js @@ -39,9 +39,19 @@ const MODULE_SUB_STORE = Object.freeze({ }, }); +const LENGTH_BYTE_ID = 32; +const LENGTH_ID = LENGTH_BYTE_ID * 2; // Each byte is represented with 2 nibbles + +const EVENT_TOPIC_PREFIX = Object.freeze({ + TX_ID: '04', + CCM_ID: '05', +}); + module.exports = { MODULE, COMMAND, EVENT, MODULE_SUB_STORE, + LENGTH_ID, + EVENT_TOPIC_PREFIX, }; diff --git a/services/export/shared/helpers/index.js b/services/export/shared/helpers/index.js index 4be31fd7e5..b80bf8ed3d 100644 --- a/services/export/shared/helpers/index.js +++ b/services/export/shared/helpers/index.js @@ -25,9 +25,18 @@ const { resolveReceivingChainID, getNetworkStatus, getUniqueChainIDs, + getBlocks, + getTransactions, } = require('./chain'); -const { MODULE, COMMAND, EVENT, MODULE_SUB_STORE } = require('./constants'); +const { + MODULE, + COMMAND, + EVENT, + MODULE_SUB_STORE, + LENGTH_ID, + EVENT_TOPIC_PREFIX, +} = require('./constants'); const { init, @@ -48,7 +57,7 @@ const { requestAppRegistry, } = require('./request'); -const { getDaysInMilliseconds, dateFromTimestamp, timeFromTimestamp } = require('./time'); +const { getToday, getDaysInMilliseconds, dateFromTimestamp, timeFromTimestamp } = require('./time'); const { normalizeTransactionAmount, @@ -66,11 +75,15 @@ module.exports = { resolveReceivingChainID, getNetworkStatus, getUniqueChainIDs, + getBlocks, + getTransactions, MODULE, COMMAND, EVENT, MODULE_SUB_STORE, + LENGTH_ID, + EVENT_TOPIC_PREFIX, init, write, @@ -87,6 +100,7 @@ module.exports = { requestConnector, requestAppRegistry, + getToday, getDaysInMilliseconds, dateFromTimestamp, timeFromTimestamp, diff --git a/services/export/shared/helpers/ready.js b/services/export/shared/helpers/ready.js new file mode 100644 index 0000000000..9a72303feb --- /dev/null +++ b/services/export/shared/helpers/ready.js @@ -0,0 +1,78 @@ +/* + * LiskHQ/lisk-service + * Copyright © 2024 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + * + */ +const Moment = require('moment'); +const MomentRange = require('moment-range'); + +const { Signals, Logger } = require('lisk-service-framework'); + +const config = require('../../config'); + +const { requestIndexer, getToday, getBlocks } = require('.'); + +const moment = MomentRange.extendMoment(Moment); +const DATE_FORMAT = config.excel.dateFormat; + +const logger = Logger(); + +let indexStatusCache; + +const getIndexStatus = async () => { + if (!indexStatusCache) { + indexStatusCache = await requestIndexer('index.status'); + + const updateIndexStatusListener = payload => { + indexStatusCache = payload; + }; + Signals.get('updateIndexStatus').add(updateIndexStatusListener); + } + return indexStatusCache; +}; + +const checkIfIndexReadyForInterval = async interval => { + try { + // Blockchain fully indexed + const { data: indexStatus } = await getIndexStatus(); + if (indexStatus.percentageIndexed === 100) return true; + + // Requested history for only until yesterday and blockchain index can already serve the information + const [, toDate] = interval.split(':'); + const to = moment(toDate, DATE_FORMAT).endOf('day'); + const today = moment(getToday(), DATE_FORMAT); + + if (to < today) { + const response = await getBlocks({ height: indexStatus.lastIndexedBlockHeight }); + const [lastIndexedBlock] = response.data; + const lastIndexedBlockGeneratedTime = lastIndexedBlock.timestamp; + + if (to <= moment(lastIndexedBlockGeneratedTime * 1000)) return true; + } + + // Allow job scheduling if the last few blocks have not been indexed yet + if (indexStatus.chainLength - indexStatus.numBlocksIndexed <= 10) { + return true; + } + } catch (err) { + logger.warn(`Index readiness check for export job scheduling failed due to: ${err.message}`); + logger.debug(err.stack); + } + + return false; +}; + +module.exports = { + getIndexStatus, + checkIfIndexReadyForInterval, +}; diff --git a/services/export/shared/helpers/time.js b/services/export/shared/helpers/time.js index 14ae71d0a3..238024d9ab 100644 --- a/services/export/shared/helpers/time.js +++ b/services/export/shared/helpers/time.js @@ -17,22 +17,28 @@ const moment = require('moment'); const config = require('../../config'); -const DAY_IN_MILLISEC = moment().endOf('day').valueOf() - moment().startOf('day').valueOf() + 1; -const getDaysInMilliseconds = days => days * DAY_IN_MILLISEC; +const DATE_FORMAT = config.excel.dateFormat; +const TIME_FORMAT = config.excel.timeFormat; + +const getToday = () => moment().format(DATE_FORMAT); + +const DAY_IN_MILLISECS = moment().endOf('day').valueOf() - moment().startOf('day').valueOf() + 1; +const getDaysInMilliseconds = days => days * DAY_IN_MILLISECS; const momentFromTimestamp = timestamp => moment.unix(timestamp); const dateFromTimestamp = timestamp => { const dateTime = momentFromTimestamp(timestamp); - return dateTime.utcOffset(0).format(config.excel.dateFormat); + return dateTime.utcOffset(0).format(DATE_FORMAT); }; const timeFromTimestamp = timestamp => { const dateTime = momentFromTimestamp(timestamp); - return dateTime.utcOffset(0).format(config.excel.timeFormat); + return dateTime.utcOffset(0).format(TIME_FORMAT); }; module.exports = { + getToday, getDaysInMilliseconds, dateFromTimestamp, timeFromTimestamp, diff --git a/services/export/shared/transactionsExport.js b/services/export/shared/transactionsExport.js index 69cfa07fc7..c0117a3121 100644 --- a/services/export/shared/transactionsExport.js +++ b/services/export/shared/transactionsExport.js @@ -25,8 +25,14 @@ const { Exceptions: { NotFoundException, ValidationException }, Queue, HTTP, + Logger, } = require('lisk-service-framework'); +const config = require('../config'); +const regex = require('./regex'); +const FilesystemCache = require('./csvCache'); +const fields = require('./excelFieldMappings'); + const { getLisk32AddressFromPublicKey, getCurrentChainID, @@ -36,9 +42,12 @@ const { COMMAND, EVENT, MODULE_SUB_STORE, + LENGTH_ID, + EVENT_TOPIC_PREFIX, requestIndexer, requestConnector, requestAppRegistry, + getToday, getDaysInMilliseconds, dateFromTimestamp, timeFromTimestamp, @@ -46,31 +55,28 @@ const { normalizeTransactionFee, checkIfSelfTokenTransfer, getUniqueChainIDs, + getBlocks, + getTransactions, } = require('./helpers'); -const config = require('../config'); -const fields = require('./excelFieldMappings'); - +const { checkIfIndexReadyForInterval } = require('./helpers/ready'); const { requestAllCustom, requestAllStandard } = require('./requestAll'); -const FilesystemCache = require('./csvCache'); -const regex = require('./regex'); const partials = FilesystemCache(config.cache.partials); const staticFiles = FilesystemCache(config.cache.exports); const noTransactionsCache = CacheRedis('noTransactions', config.endpoints.volatileRedis); +const jobScheduledCache = CacheRedis('jobScheduled', config.endpoints.volatileRedis); const DATE_FORMAT = config.excel.dateFormat; -const MAX_NUM_TRANSACTIONS = 10000; +const MAX_NUM_TRANSACTIONS = 10e5; + +const logger = Logger(); -let tokenModuleData; let feeTokenID; let defaultStartDate; - -const getTransactions = async params => requestIndexer('transactions', params); -const getBlocks = async params => requestIndexer('blocks', params); - -const getGenesisBlock = async height => requestIndexer('blocks', { height }); +let tokenModuleData; +let loadingAssets = false; const getAddressFromParams = params => params.address || getLisk32AddressFromPublicKey(params.publicKey); @@ -85,8 +91,9 @@ const getTransactionsInAsc = async params => }); const validateIfAccountExists = async address => { - const { data: tokenBalances } = await requestIndexer('token.balances', { address }); - return !!tokenBalances.length; + const response = await requestIndexer('token.account.exists', { address }); + const { isExists } = response.data; + return isExists; }; const getEvents = async params => @@ -112,7 +119,11 @@ const getCrossChainTransferTransactionInfo = async params => { for (let i = 0; i < ccmTransferEvents.length; i++) { const ccmTransferEvent = ccmTransferEvents[i]; - const [ccuTransactionID] = ccmTransferEvent.topics; + const [ccuTransactionIDTopic] = ccmTransferEvent.topics; + const ccuTransactionID = + ccuTransactionIDTopic.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID + ? ccuTransactionIDTopic.slice(EVENT_TOPIC_PREFIX.TX_ID.length) + : ccuTransactionIDTopic; const [transaction] = (await requestIndexer('transactions', { id: ccuTransactionID })).data; transactions.push({ id: ccuTransactionID, @@ -145,7 +156,11 @@ const getRewardAssignedInfo = async params => { for (let i = 0; i < rewardsAssignedEvents.length; i++) { const rewardsAssignedEvent = rewardsAssignedEvents[i]; - const [transactionID] = rewardsAssignedEvent.topics; + const [transactionIDTopic] = rewardsAssignedEvent.topics; + const transactionID = + transactionIDTopic.length === EVENT_TOPIC_PREFIX.TX_ID.length + LENGTH_ID + ? transactionIDTopic.slice(EVENT_TOPIC_PREFIX.TX_ID.length) + : transactionIDTopic; const [transaction] = (await requestIndexer('transactions', { id: transactionID })).data; transactions.push({ @@ -217,27 +232,51 @@ const getChainInfo = async chainID => { return { chainID, chainName }; }; -const getOpeningBalance = async address => { - if (!tokenModuleData) { - const genesisBlockAssetsLength = await requestConnector('getGenesisAssetsLength', { +const getTokenBalancesAtGenesis = async () => { + if (!tokenModuleData && !loadingAssets) { + loadingAssets = true; // loadingAssets avoids repeated invocations + + // Asynchronously fetch the token module genesis assets and cache locally + logger.info('Attempting to fetch and cache the token module genesis assets.'); + requestConnector('getGenesisAssetsLength', { module: MODULE.TOKEN, subStore: MODULE_SUB_STORE.TOKEN.USER, - }); - const totalUsers = genesisBlockAssetsLength[MODULE.TOKEN][MODULE_SUB_STORE.TOKEN.USER]; - - tokenModuleData = ( - await requestAllCustom( - requestConnector, - 'getGenesisAssetByModule', - { module: MODULE.TOKEN, subStore: MODULE_SUB_STORE.TOKEN.USER }, - totalUsers, - ) - ).userSubstore; + }) + .then(async genesisBlockAssetsLength => { + const totalUsers = genesisBlockAssetsLength[MODULE.TOKEN][MODULE_SUB_STORE.TOKEN.USER]; + + const response = await requestAllCustom( + requestConnector, + 'getGenesisAssetByModule', + { module: MODULE.TOKEN, subStore: MODULE_SUB_STORE.TOKEN.USER, limit: 1000 }, + totalUsers, + ); + + tokenModuleData = response[MODULE_SUB_STORE.TOKEN.USER]; + loadingAssets = false; + logger.info('Successfully cached token module genesis assets.'); + }) + .catch(err => { + logger.warn( + `Failed to fetch token module genesis assets. Will retry later.\nError: ${err.message}`, + ); + logger.debug(err.stack); + + loadingAssets = false; + }); } - const filteredAccount = tokenModuleData.find(e => e.address === address); - const openingBalance = filteredAccount - ? { tokenID: filteredAccount.tokenID, amount: filteredAccount.availableBalance } + return tokenModuleData; +}; + +const getOpeningBalance = async address => { + const balancesAtGenesis = await getTokenBalancesAtGenesis(); + const accountInfo = balancesAtGenesis + ? balancesAtGenesis.find(e => e.address === address) + : await requestConnector('getTokenBalanceAtGenesis', { address }); + + const openingBalance = accountInfo + ? { tokenID: accountInfo.tokenID, amount: accountInfo.availableBalance } : null; return openingBalance; @@ -269,15 +308,13 @@ const getDefaultStartDate = async () => { } = await getNetworkStatus(); const { data: [block], - } = await getGenesisBlock(genesisHeight); + } = await getBlocks({ height: genesisHeight }); defaultStartDate = moment(block.timestamp * 1000).format(DATE_FORMAT); } return defaultStartDate; }; -const getToday = () => moment().format(DATE_FORMAT); - const standardizeIntervalFromParams = async ({ interval }) => { let from; let to; @@ -377,7 +414,29 @@ const normalizeTransaction = (address, tx, currentChainID, txFeeTokenID) => { }; }; +const rescheduleOnTimeout = async params => { + try { + const currentChainID = await getCurrentChainID(); + const excelFilename = await getExcelFilenameFromParams(params, currentChainID); + + // Clear the flag to allow proper execution on user request if auto re-scheduling fails + await jobScheduledCache.delete(excelFilename); + + const { address } = params; + const requestInterval = await standardizeIntervalFromParams(params); + logger.info(`Original job timed out. Re-scheduling job for ${address} (${requestInterval}).`); + + // eslint-disable-next-line no-use-before-define + await scheduleTransactionExportQueue.add({ params }); + } catch (err) { + logger.warn(`History export job Re-scheduling failed due to: ${err.message}`); + logger.debug(err.stack); + } +}; + const exportTransactions = async job => { + let timeout; + const { params } = job.data; const allTransactions = []; @@ -386,59 +445,77 @@ const exportTransactions = async job => { // Validate if account has transactions const isAccountHasTransactions = await validateIfAccountHasTransactions(params.address); if (isAccountHasTransactions) { + // Add a timeout to automatically re-schedule, if the current job run times out on the last attempt + if (job.attemptsMade === job.opts.attempts - 1) { + timeout = setTimeout( + rescheduleOnTimeout.bind(null, params), + config.queue.scheduleTransactionExport.options.defaultJobOptions.timeout, + ); + } + const interval = await standardizeIntervalFromParams(params); const [from, to] = interval.split(':'); const range = moment.range(moment(from, DATE_FORMAT), moment(to, DATE_FORMAT)); const arrayOfDates = Array.from(range.by('day')).map(d => d.format(DATE_FORMAT)); - for (let i = 0; i < arrayOfDates.length; i++) { - const day = arrayOfDates[i]; + // eslint-disable-next-line no-restricted-syntax + for (const day of arrayOfDates) { const partialFilename = await getPartialFilenameFromParams(params, day); + + // No history available for the specified day + if ((await noTransactionsCache.get(partialFilename)) === true) { + // eslint-disable-next-line no-continue + continue; + } + + // History available as a partial file for the specified day if (await partials.fileExists(partialFilename)) { const transactions = JSON.parse(await partials.read(partialFilename)); allTransactions.push(...transactions); - } else if ((await noTransactionsCache.get(partialFilename)) !== true) { - const fromTimestamp = moment(day, DATE_FORMAT).startOf('day').unix(); - const toTimestamp = moment(day, DATE_FORMAT).endOf('day').unix(); - const timestampRange = `${fromTimestamp}:${toTimestamp}`; - const transactions = await requestAllStandard( - getTransactionsInAsc, - { ...params, timestamp: timestampRange }, - MAX_NUM_TRANSACTIONS, - ); - allTransactions.push(...transactions); - const incomingCrossChainTransferTxs = await getCrossChainTransferTransactionInfo({ - ...params, - timestamp: timestampRange, - }); - - const blocks = await getBlocksInAsc({ - ...params, - timestamp: timestampRange, - }); - allBlocks.push(...blocks); - - const rewardAssignedInfo = await getRewardAssignedInfo({ - ...params, - timestamp: timestampRange, - }); - - if (incomingCrossChainTransferTxs.length || rewardAssignedInfo.length) { - allTransactions.push(...incomingCrossChainTransferTxs, ...rewardAssignedInfo); - allTransactions.sort((a, b) => a.block.height - b.block.height); - } + // eslint-disable-next-line no-continue + continue; + } + + // Query for history and build the partial + const fromTimestamp = moment(day, DATE_FORMAT).startOf('day').unix(); + const toTimestamp = moment(day, DATE_FORMAT).endOf('day').unix(); + const timestampRange = `${fromTimestamp}:${toTimestamp}`; + const transactions = await requestAllStandard( + getTransactionsInAsc, + { ...params, timestamp: timestampRange }, + MAX_NUM_TRANSACTIONS, + ); + allTransactions.push(...transactions); + + const incomingCrossChainTransferTxs = await getCrossChainTransferTransactionInfo({ + ...params, + timestamp: timestampRange, + }); + + const blocks = await getBlocksInAsc({ + ...params, + timestamp: timestampRange, + }); + allBlocks.push(...blocks); + + const rewardAssignedInfo = await getRewardAssignedInfo({ + ...params, + timestamp: timestampRange, + }); + + if (incomingCrossChainTransferTxs.length || rewardAssignedInfo.length) { + allTransactions.push(...incomingCrossChainTransferTxs, ...rewardAssignedInfo); + allTransactions.sort((a, b) => a.block.height - b.block.height); + } - if (day !== getToday()) { - if (transactions.length) { - partials.write(partialFilename, JSON.stringify(allTransactions)); - } else { - // Flag to prevent unnecessary calls to core/storage space usage on the file cache - const RETENTION_PERIOD_MS = getDaysInMilliseconds( - config.cache.partials.retentionInDays, - ); - await noTransactionsCache.set(partialFilename, true, RETENTION_PERIOD_MS); - } + if (day !== getToday()) { + if (transactions.length) { + partials.write(partialFilename, JSON.stringify(allTransactions)); + } else { + // Flag to prevent unnecessary calls to the node/file cache + const RETENTION_PERIOD_MS = getDaysInMilliseconds(config.cache.partials.retentionInDays); + await noTransactionsCache.set(partialFilename, true, RETENTION_PERIOD_MS); } } } @@ -478,6 +555,10 @@ const exportTransactions = async job => { metadataSheet.addRows(metadata); await workBook.xlsx.writeFile(`${config.cache.exports.dirPath}/${excelFilename}`); + await jobScheduledCache.delete(excelFilename); // Remove the entry from cache to free up memory + + // Clear the auto re-schedule timeout on successful completion + clearTimeout(timeout); }; const scheduleTransactionExportQueue = Queue( @@ -485,14 +566,10 @@ const scheduleTransactionExportQueue = Queue( config.queue.scheduleTransactionExport.name, exportTransactions, config.queue.scheduleTransactionExport.concurrency, + config.queue.scheduleTransactionExport.options, ); const scheduleTransactionHistoryExport = async params => { - // Schedule only when index is completely built - const isBlockchainIndexReady = await requestIndexer('isBlockchainFullyIndexed'); - if (!isBlockchainIndexReady) - throw new ValidationException('The blockchain index is not yet ready. Please retry later.'); - const exportResponse = { data: {}, meta: { @@ -502,26 +579,50 @@ const scheduleTransactionHistoryExport = async params => { const { publicKey } = params; const address = getAddressFromParams(params); - - // Validate if account exists - const isAccountExists = await validateIfAccountExists(address); - if (!isAccountExists) throw new NotFoundException(`Account ${address} not found.`); + const requestInterval = await standardizeIntervalFromParams(params); exportResponse.data.address = address; exportResponse.data.publicKey = publicKey; - exportResponse.data.interval = await standardizeIntervalFromParams(params); + exportResponse.data.interval = requestInterval; const currentChainID = await getCurrentChainID(); const excelFilename = await getExcelFilenameFromParams(params, currentChainID); + + // Job already scheduled, skip remaining checks + if ((await jobScheduledCache.get(excelFilename)) === true) { + return exportResponse; + } + + // Request already processed and the history is ready to be downloaded if (await staticFiles.fileExists(excelFilename)) { exportResponse.data.fileName = excelFilename; exportResponse.data.fileUrl = await getExcelFileUrlFromParams(params, currentChainID); exportResponse.meta.ready = true; - } else { - await scheduleTransactionExportQueue.add({ params: { ...params, address } }); - exportResponse.status = 'ACCEPTED'; + + return exportResponse; } + // Validate if account exists + const isAccountExists = await validateIfAccountExists(address); + if (!isAccountExists) throw new NotFoundException(`Account ${address} not found.`); + + // Validate if the index is ready enough to serve the user request + const isBlockchainIndexReady = await checkIfIndexReadyForInterval(requestInterval); + if (!isBlockchainIndexReady) { + throw new ValidationException( + `The blockchain index is not yet ready for the requested interval (${requestInterval}). Please retry later.`, + ); + } + + // Schedule a new job to process the history export + await scheduleTransactionExportQueue.add({ params: { ...params, address } }); + exportResponse.status = 'ACCEPTED'; + + const ttl = + config.queue.scheduleTransactionExport.options.defaultJobOptions.timeout * + config.queue.scheduleTransactionExport.options.defaultJobOptions.attempts; + await jobScheduledCache.set(excelFilename, true, ttl); + return exportResponse; }; @@ -555,6 +656,7 @@ module.exports = { exportTransactions, scheduleTransactionHistoryExport, downloadTransactionHistory, + getTokenBalancesAtGenesis, // For functional tests getAddressFromParams, @@ -571,4 +673,5 @@ module.exports = { getMetadata, resolveChainIDs, normalizeBlocks, + validateIfAccountExists, }; diff --git a/services/export/tests/unit/shared/helpers/ready.test.js b/services/export/tests/unit/shared/helpers/ready.test.js new file mode 100644 index 0000000000..b405d41a4e --- /dev/null +++ b/services/export/tests/unit/shared/helpers/ready.test.js @@ -0,0 +1,356 @@ +/* + * LiskHQ/lisk-service + * Copyright © 2024 Lisk Foundation + * + * See the LICENSE file at the top-level directory of this distribution + * for licensing information. + * + * Unless otherwise agreed in a custom licensing agreement with the Lisk Foundation, + * no part of this software, including this file, may be copied, modified, + * propagated, or distributed except according to the terms contained in the + * LICENSE file. + * + * Removal or modification of this copyright notice is prohibited. + * + */ +const { resolve } = require('path'); + +const mockedRequestFilePath = resolve(`${__dirname}/../../../../shared/helpers/request`); +const mockedHelpersPath = resolve(`${__dirname}/../../../../shared/helpers`); + +describe('Test getIndexStatus method', () => { + it('should return the index status', async () => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23827824, + lastIndexedBlockHeight: 23395523, + chainLength: 436833, + numBlocksIndexed: 4532, + percentageIndexed: 1.03, + isIndexingInProgress: true, + }, + meta: { + lastUpdate: 1706251273, + }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + const { getIndexStatus } = require('../../../../shared/helpers/ready'); + const response = await getIndexStatus(); + + expect(response).toEqual(mockIndexStatus); + }); +}); + +describe('Test checkIfIndexReadyForInterval method', () => { + beforeEach(() => jest.resetModules()); + + describe('when indexing is at 100 percent', () => { + beforeEach(() => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23827887, + lastIndexedBlockHeight: 23827887, + chainLength: 436896, + numBlocksIndexed: 436896, + percentageIndexed: 100, + isIndexingInProgress: false, + }, + meta: { + lastUpdate: 1706251915, + }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + }); + + it('should return true interval is the day of re-genesis', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2023-12-05'); + expect(got).toEqual(want); + }); + + it('should return true interval starts on the day of re-genesis and is in the future', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2023-12-31'); + expect(got).toEqual(want); + }); + + it('should return true interval starts before the day of re-genesis and is in the future', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-04:2023-12-31'); + expect(got).toEqual(want); + }); + }); + + describe('when indexing is below 100 percent', () => { + beforeEach(() => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23827963, + lastIndexedBlockHeight: 23631326, + chainLength: 436972, + numBlocksIndexed: 15523, + percentageIndexed: 55, + isIndexingInProgress: true, + }, + meta: { lastUpdate: 1706252683 }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + jest.mock(mockedHelpersPath, () => { + const actual = jest.requireActual(mockedHelpersPath); + return { + ...actual, + getToday() { + return '2024-01-26'; + }, + getBlocks() { + // timestamp for block at height 23631326: Wednesday, January 3, 2024 7:59:20 AM + return { data: [{ timestamp: 1704268760 }] }; + }, + }; + }); + }); + + it('should return true when interval ends before the lastIndexedBlockHeight timestamp', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2023-12-31'); + expect(got).toEqual(want); + }); + + it('should return false when interval ends the same day but after the lastIndexedBlockHeight timestamp', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = false; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2024-01-03'); + expect(got).toEqual(want); + }); + + it('should return false when interval ends after the lastIndexedBlockHeight timestamp', async () => { + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = false; + + const got = await checkIfIndexReadyForInterval('2023-12-04:2024-01-26'); + expect(got).toEqual(want); + }); + + it('should return false when getBlocks does not return proper response', async () => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23827963, + lastIndexedBlockHeight: 23631326, + chainLength: 436972, + numBlocksIndexed: 15523, + percentageIndexed: 55, + isIndexingInProgress: true, + }, + meta: { lastUpdate: 1706252683 }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + jest.mock(mockedHelpersPath, () => { + const actual = jest.requireActual(mockedHelpersPath); + return { + ...actual, + getToday() { + return '2024-01-26'; + }, + getBlocks() { + return { error: true, message: 'mocked error' }; + }, + }; + }); + + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = false; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2023-12-31'); + expect(got).toEqual(want); + }); + }); + + describe('when indexing is around 99.99 percent', () => { + it('should return true when interval ends today and the indexing lags by 2 blocks', async () => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23828082, + lastIndexedBlockHeight: 23828080, + chainLength: 437091, + numBlocksIndexed: 437089, + percentageIndexed: 99.99, + isIndexingInProgress: false, + }, + meta: { lastUpdate: 1706253875 }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + jest.mock(mockedHelpersPath, () => { + const actual = jest.requireActual(mockedHelpersPath); + return { + ...actual, + getToday() { + return '2024-01-26'; + }, + getBlocks() { + // timestamp for block at height 23828080: Friday, January 26, 2024 7:24:10 AM + return { data: [{ timestamp: 1706253850 }] }; + }, + }; + }); + + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2024-01-26'); + expect(got).toEqual(want); + }); + + it('should return true when interval ends today and the indexing lags by 10 blocks', async () => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23828082, + lastIndexedBlockHeight: 23828072, + chainLength: 437091, + numBlocksIndexed: 437081, + percentageIndexed: 99.99, + isIndexingInProgress: false, + }, + meta: { lastUpdate: 1706253875 }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + jest.mock(mockedHelpersPath, () => { + const actual = jest.requireActual(mockedHelpersPath); + return { + ...actual, + getToday() { + return '2024-01-26'; + }, + getBlocks() { + // timestamp for block at height 23828072: Friday, January 26, 2024 7:22:50 AM + return { data: [{ timestamp: 1706253770 }] }; + }, + }; + }); + + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = true; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2024-01-26'); + expect(got).toEqual(want); + }); + + it('should return false when interval ends today and the indexing lags by 100 blocks', async () => { + const mockIndexStatus = { + data: { + genesisHeight: 23390992, + lastBlockHeight: 23828082, + lastIndexedBlockHeight: 23827982, + chainLength: 437091, + numBlocksIndexed: 436991, + percentageIndexed: 99.97, + isIndexingInProgress: false, + }, + meta: { lastUpdate: 1706253875 }, + }; + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return mockIndexStatus; + }, + }; + }); + + jest.mock(mockedHelpersPath, () => { + const actual = jest.requireActual(mockedHelpersPath); + return { + ...actual, + getToday() { + return '2024-01-26'; + }, + getBlocks() { + // timestamp for block at height 23827982: Friday, January 26, 2024 7:07:50 AM + return { data: [{ timestamp: 1706252870 }] }; + }, + }; + }); + + const { checkIfIndexReadyForInterval } = require('../../../../shared/helpers/ready'); + const want = false; + + const got = await checkIfIndexReadyForInterval('2023-12-05:2024-01-26'); + expect(got).toEqual(want); + }); + }); +}); diff --git a/services/export/tests/unit/shared/transactionsExport.test.js b/services/export/tests/unit/shared/transactionsExport.test.js index 448c2a8220..8b417745ce 100644 --- a/services/export/tests/unit/shared/transactionsExport.test.js +++ b/services/export/tests/unit/shared/transactionsExport.test.js @@ -14,7 +14,7 @@ * */ /* eslint-disable mocha/max-top-level-suites */ - +/* eslint-disable import/no-dynamic-require */ const { resolve } = require('path'); const moment = require('moment'); @@ -72,34 +72,16 @@ const partialFilenameExtension = '.json'; describe('Test getOpeningBalance method', () => { it('should return opening balance when called with valid address', async () => { - const mockUserSubstore = [ - { - address: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', - availableBalance: '100000000000000', - lockedBalances: [], - tokenID: '0400000000000000', - }, - ]; - - jest.mock(mockedRequestAllFilePath, () => { - const actual = jest.requireActual(mockedRequestAllFilePath); - return { - ...actual, - requestAllCustom() { - return { userSubstore: mockUserSubstore }; - }, - }; - }); + const mockUserSubstore = { + address: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + availableBalance: '100000000000000', + lockedBalances: [], + tokenID: '0400000000000000', + }; - jest.mock(mockedRequestFilePath, () => { - const actual = jest.requireActual(mockedRequestFilePath); - return { - ...actual, - requestConnector() { - return { token: { userSubstore: mockUserSubstore } }; - }, - }; - }); + jest.mock(mockedRequestFilePath); + const { requestConnector } = require(mockedRequestFilePath); + requestConnector.mockResolvedValueOnce(undefined).mockResolvedValueOnce(mockUserSubstore); const { getOpeningBalance } = require('../../../shared/transactionsExport'); @@ -129,7 +111,95 @@ describe('Test getOpeningBalance method', () => { }); describe('Test getCrossChainTransferTransactionInfo method', () => { - it('should return transaction info when called with valid address', async () => { + it('should return transaction info when called with valid address (event topic contains transaction prefix)', async () => { + const mockEventData = [ + { + id: 'efe94d3a5ad35297098614100c5dd7bff6657d38baed08fb850fa9ce69b0862c', + module: 'token', + name: 'ccmTransfer', + data: { + senderAddress: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + recipientAddress: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + tokenID: '0400000000000000', + amount: '100000000000', + receivingChainID: '04000001', + result: 0, + }, + topics: [ + '04efcbab90c4769dc47029412010ef76623722678f446a7417f59fed998a6407de', + 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + ], + block: { + id: '1fc7e1a4a06a6b9610ed5e4fb48c9f839b1fcd0f91b3f6d4c22f9f64eac40657', + height: 313, + timestamp: 1689693410, + }, + }, + ]; + + jest.mock(mockedRequestAllFilePath, () => { + const actual = jest.requireActual(mockedRequestAllFilePath); + return { + ...actual, + requestAllStandard() { + return mockEventData; + }, + }; + }); + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return { + data: [ + { + moduleCommand: 'interoperability:submitSidechainCrossChainUpdate', + params: { sendingChainID: '04000000' }, + }, + ], + }; + }, + }; + }); + + const { getCrossChainTransferTransactionInfo } = require('../../../shared/transactionsExport'); + + const crossChainTransferTxs = await getCrossChainTransferTransactionInfo({ + address: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + }); + const expectedResponse = [ + { + block: { + id: '1fc7e1a4a06a6b9610ed5e4fb48c9f839b1fcd0f91b3f6d4c22f9f64eac40657', + height: 313, + timestamp: 1689693410, + }, + id: 'efcbab90c4769dc47029412010ef76623722678f446a7417f59fed998a6407de', + isIncomingCrossChainTransferTransaction: true, + moduleCommand: 'interoperability:submitSidechainCrossChainUpdate', + params: { + amount: '100000000000', + data: "This entry was generated from 'ccmTransfer' event emitted from the specified CCU transactionID.", + receivingChainID: '04000001', + recipientAddress: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + result: 0, + senderAddress: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + tokenID: '0400000000000000', + }, + sender: { + address: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + }, + sendingChainID: '04000000', + }, + ]; + + expect(crossChainTransferTxs).toEqual(expectedResponse); + }); + + it('should return transaction info when called with valid address (event topic does not contain transaction prefix)', async () => { const mockEventData = [ { id: 'efe94d3a5ad35297098614100c5dd7bff6657d38baed08fb850fa9ce69b0862c', @@ -234,7 +304,103 @@ describe('Test getCrossChainTransferTransactionInfo method', () => { }); describe('Test getRewardAssignedInfo method', () => { - it('should return reward assigned info when called with valid address', async () => { + it('should return reward assigned info when called with valid address (event topic contains transaction prefix)', async () => { + const mockEventData = [ + { + id: 'efe94d3a5ad35297098614100c5dd7bff6657d38baed08fb850fa9ce69b0862c', + module: 'pos', + name: 'rewardsAssigned', + data: { + stakerAddress: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + validatorAddress: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + tokenID: '0400000000000000', + amount: '100000000000', + result: 0, + }, + topics: [ + '04efcbab90c4769dc47029412010ef76623722678f446a7417f59fed998a6407de', + 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + ], + block: { + id: '1fc7e1a4a06a6b9610ed5e4fb48c9f839b1fcd0f91b3f6d4c22f9f64eac40657', + height: 313, + timestamp: 1689693410, + }, + }, + ]; + + jest.mock(mockedRequestAllFilePath, () => { + const actual = jest.requireActual(mockedRequestAllFilePath); + return { + ...actual, + requestAllStandard() { + return mockEventData; + }, + }; + }); + + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return { + data: [ + { + moduleCommand: 'pos:stake', + params: { + stakes: [ + { + validatorAddress: 'lskkdvzyxhvm2kmgs8hmteaad2zrjbjmf4cft9zpp', + amount: '-1000000000', + }, + { + validatorAddress: 'lsk64zamp63e9km9p6vtfea9c5pda2wuw79tc8a9k', + amount: '2000000000', + }, + ], + }, + }, + ], + }; + }, + }; + }); + + const { getRewardAssignedInfo } = require('../../../shared/transactionsExport'); + + const rewardsAssignedInfo = await getRewardAssignedInfo({ + address: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + }); + const expectedResponse = [ + { + block: { + id: '1fc7e1a4a06a6b9610ed5e4fb48c9f839b1fcd0f91b3f6d4c22f9f64eac40657', + height: 313, + timestamp: 1689693410, + }, + id: 'efcbab90c4769dc47029412010ef76623722678f446a7417f59fed998a6407de', + moduleCommand: 'pos:stake', + params: { + amount: '100000000000', + data: "This entry was generated from 'rewardsAssigned' event emitted from the specified transactionID.", + validatorAddress: 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + result: 0, + stakerAddress: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + tokenID: '0400000000000000', + }, + sender: { + address: 'lskguo9kqnea2zsfo3a6qppozsxsg92nuuma3p7ad', + }, + rewardAmount: '100000000000', + rewardTokenID: '0400000000000000', + }, + ]; + + expect(rewardsAssignedInfo).toEqual(expectedResponse); + }); + + it('should return reward assigned info when called with valid address (event topic does not contain transaction prefix)', async () => { const mockEventData = [ { id: 'efe94d3a5ad35297098614100c5dd7bff6657d38baed08fb850fa9ce69b0862c', @@ -453,3 +619,27 @@ describe('Test normalizeBlocks method', () => { expect(normalizeBlocks(undefined)).rejects.toThrow(); }); }); + +describe('Test validateIfAccountExists method', () => { + it('should return true when account exists', async () => { + jest.mock(mockedRequestFilePath, () => { + const actual = jest.requireActual(mockedRequestFilePath); + return { + ...actual, + requestIndexer() { + return { + data: { isExists: true }, + meta: {}, + }; + }, + }; + }); + + const { validateIfAccountExists } = require('../../../shared/transactionsExport'); + + const isAccountExists = await validateIfAccountExists( + 'lskyvvam5rxyvbvofxbdfcupxetzmqxu22phm4yuo', + ); + expect(isAccountExists).toEqual(true); + }); +}); diff --git a/services/gateway/apis/http-exports/swagger/definitions/export.json b/services/gateway/apis/http-exports/swagger/definitions/export.json index f214994903..363e8e88a2 100644 --- a/services/gateway/apis/http-exports/swagger/definitions/export.json +++ b/services/gateway/apis/http-exports/swagger/definitions/export.json @@ -72,13 +72,13 @@ "fileName": { "type": "string", "format": "fileName", - "example": "transactions__
__.csv", + "example": "transactions__
__.xlsx", "description": "The name of the file containing the exported account transaction history.\n" }, "fileUrl": { "type": "string", "format": "fileUrl", - "example": "/api/v3/exports/transactions__
__.csv", + "example": "/api/v3/exports/transactions__
__.xlsx", "description": "The file URL path containing the exported account transaction history.\n" } } diff --git a/services/gateway/tests/constants/utils.js b/services/gateway/tests/constants/utils.js index 25554f485b..f1f6ad9b43 100644 --- a/services/gateway/tests/constants/utils.js +++ b/services/gateway/tests/constants/utils.js @@ -138,14 +138,14 @@ const requireAllJsonExpectedResponse = { fileName: { type: 'string', format: 'fileName', - example: 'transactions__
__.csv', + example: 'transactions__
__.xlsx', description: 'The name of the file containing the exported account transaction history.\n', }, fileUrl: { type: 'string', format: 'fileUrl', - example: '/api/v3/exports/transactions__
__.csv', + example: '/api/v3/exports/transactions__
__.xlsx', description: 'The file URL path containing the exported account transaction history.\n', }, }, diff --git a/services/transaction-statistics/shared/buildTransactionStatistics.js b/services/transaction-statistics/shared/buildTransactionStatistics.js index b42e84a3d2..5f4c78d9e6 100644 --- a/services/transaction-statistics/shared/buildTransactionStatistics.js +++ b/services/transaction-statistics/shared/buildTransactionStatistics.js @@ -198,7 +198,7 @@ const fetchTransactionsForPastNDays = async (n, forceReload = false) => { i === 0 || !(await transactionStatisticsTable.find({ date, limit: 1 }, ['id'])).length; if (shouldUpdate || forceReload) { - const formattedDate = moment.unix(date).format('YYYY-MM-DD'); + const formattedDate = moment.unix(date).format(DATE_FORMAT.DAY); logger.debug(`Adding day ${i + 1}, ${formattedDate} to the queue.`); await transactionStatisticsQueue.add({ date }); logger.info(`Added day ${i + 1}, ${formattedDate} to the queue.`);