diff --git a/alm/src/components/ConfirmationsContainer.tsx b/alm/src/components/ConfirmationsContainer.tsx index 57ff0b7d0..c9a37ab9a 100644 --- a/alm/src/components/ConfirmationsContainer.tsx +++ b/alm/src/components/ConfirmationsContainer.tsx @@ -39,10 +39,17 @@ export interface ConfirmationsContainerParams { message: MessageObject receipt: Maybe fromHome: boolean - timestamp: number + homeStartBlock: Maybe + foreignStartBlock: Maybe } -export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp }: ConfirmationsContainerParams) => { +export const ConfirmationsContainer = ({ + message, + receipt, + fromHome, + homeStartBlock, + foreignStartBlock +}: ConfirmationsContainerParams) => { const { home: { name: homeName }, foreign: { name: foreignName } @@ -62,7 +69,8 @@ export const ConfirmationsContainer = ({ message, receipt, fromHome, timestamp } message, receipt, fromHome, - timestamp, + homeStartBlock, + foreignStartBlock, requiredSignatures, validatorList, blockConfirmations diff --git a/alm/src/components/ExecutionConfirmation.tsx b/alm/src/components/ExecutionConfirmation.tsx index 1541e7bf6..0ee1ff880 100644 --- a/alm/src/components/ExecutionConfirmation.tsx +++ b/alm/src/components/ExecutionConfirmation.tsx @@ -36,10 +36,12 @@ export const ExecutionConfirmation = ({ const availableManualExecution = !isHome && (executionData.status === VALIDATOR_CONFIRMATION_STATUS.WAITING || + executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED || (executionData.status === VALIDATOR_CONFIRMATION_STATUS.UNDEFINED && executionEventsFetched && !!executionData.validator)) const requiredManualExecution = availableManualExecution && ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION + const showAgeColumn = !requiredManualExecution || executionData.status === VALIDATOR_CONFIRMATION_STATUS.FAILED const windowWidth = useWindowWidth() const txExplorerLink = getExplorerTxUrl(executionData.txHash, isHome) @@ -71,7 +73,7 @@ export const ExecutionConfirmation = ({ {requiredManualExecution ? 'Execution info' : 'Executed by'} Status - {!requiredManualExecution && Age} + {showAgeColumn && Age} {availableManualExecution && Actions} @@ -87,7 +89,7 @@ export const ExecutionConfirmation = ({ )} {getExecutionStatusElement(executionData.status)} - {!requiredManualExecution && ( + {showAgeColumn && ( {executionData.timestamp > 0 ? ( diff --git a/alm/src/components/ManualExecutionButton.tsx b/alm/src/components/ManualExecutionButton.tsx index e39c61506..5a7d1f90a 100644 --- a/alm/src/components/ManualExecutionButton.tsx +++ b/alm/src/components/ManualExecutionButton.tsx @@ -2,9 +2,17 @@ import React, { useState, useEffect } from 'react' import styled from 'styled-components' import { InjectedConnector } from '@web3-react/injected-connector' import { useWeb3React } from '@web3-react/core' -import { INCORRECT_CHAIN_ERROR, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' +import { + DOUBLE_EXECUTION_ATTEMPT_ERROR, + EXECUTION_FAILED_ERROR, + EXECUTION_OUT_OF_GAS_ERROR, + INCORRECT_CHAIN_ERROR, + VALIDATOR_CONFIRMATION_STATUS +} from '../config/constants' import { useStateProvider } from '../state/StateProvider' import { signatureToVRS, packSignatures } from '../utils/signatures' +import { getSuccessExecutionData } from '../utils/getFinalizationEvent' +import { TransactionReceipt } from 'web3-eth' const StyledButton = styled.button` color: var(--button-color); @@ -61,7 +69,9 @@ export const ManualExecutionButton = ({ if (!library || !foreign.bridgeContract || !signatureCollected || !signatureCollected.length) return const signatures = packSignatures(signatureCollected.map(signatureToVRS)) - const data = foreign.bridgeContract.methods.executeSignatures(messageData, signatures).encodeABI() + const messageId = messageData.slice(0, 66) + const bridge = foreign.bridgeContract + const data = bridge.methods.executeSignatures(messageData, signatures).encodeABI() setManualExecution(false) library.eth @@ -80,7 +90,27 @@ export const ManualExecutionButton = ({ }) setPendingExecution(true) }) - .on('error', (e: Error) => setError(e.message)) + .on('error', async (e: Error, receipt: TransactionReceipt) => { + if (e.message.includes('Transaction has been reverted by the EVM')) { + const successExecutionData = await getSuccessExecutionData(bridge, 'RelayedMessage', library, messageId) + if (successExecutionData) { + setExecutionData(successExecutionData) + setError(DOUBLE_EXECUTION_ATTEMPT_ERROR) + } else { + const { gas } = await library.eth.getTransaction(receipt.transactionHash) + setExecutionData({ + status: VALIDATOR_CONFIRMATION_STATUS.FAILED, + validator: account, + txHash: receipt.transactionHash, + timestamp: Math.floor(new Date().getTime() / 1000.0), + executionResult: false + }) + setError(gas === receipt.gasUsed ? EXECUTION_OUT_OF_GAS_ERROR : EXECUTION_FAILED_ERROR) + } + } else { + setError(e.message) + } + }) }, [ manualExecution, diff --git a/alm/src/components/StatusContainer.tsx b/alm/src/components/StatusContainer.tsx index c5d9e9efe..83c4bdee5 100644 --- a/alm/src/components/StatusContainer.tsx +++ b/alm/src/components/StatusContainer.tsx @@ -10,6 +10,7 @@ import { ExplorerTxLink } from './commons/ExplorerTxLink' import { ConfirmationsContainer } from './ConfirmationsContainer' import { TransactionReceipt } from 'web3-eth' import { BackButton } from './commons/BackButton' +import { useClosestBlock } from '../hooks/useClosestBlock' export interface StatusContainerParam { onBackToMain: () => void @@ -23,12 +24,15 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar const { chainId, txHash, messageIdParam } = useParams() const validChainId = chainId === home.chainId.toString() || chainId === foreign.chainId.toString() const validParameters = validChainId && validTxHash(txHash) + const isHome = chainId === home.chainId.toString() const { messages, receipt, status, description, timestamp, loading } = useTransactionStatus({ txHash: validParameters ? txHash : '', chainId: validParameters ? parseInt(chainId) : 0, receiptParam }) + const homeStartBlock = useClosestBlock(true, isHome, receipt, timestamp) + const foreignStartBlock = useClosestBlock(false, isHome, receipt, timestamp) const selectedMessageId = messageIdParam === undefined || messages[messageIdParam] === undefined ? -1 : messageIdParam @@ -64,7 +68,6 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar const displayReference = multiMessageSelected ? messages[selectedMessageId].id : txHash const formattedMessageId = formatTxHash(displayReference) - const isHome = chainId === home.chainId.toString() const txExplorerLink = getExplorerTxUrl(txHash, isHome) const displayExplorerLink = status !== TRANSACTION_STATUS.NOT_FOUND @@ -101,7 +104,13 @@ export const StatusContainer = ({ onBackToMain, setNetworkFromParams, receiptPar )} {displayMessageSelector && } {displayConfirmations && ( - + )} diff --git a/alm/src/components/commons/ErrorAlert.tsx b/alm/src/components/commons/ErrorAlert.tsx index af3e3f598..236353b00 100644 --- a/alm/src/components/commons/ErrorAlert.tsx +++ b/alm/src/components/commons/ErrorAlert.tsx @@ -2,6 +2,7 @@ import React from 'react' import styled from 'styled-components' import { InfoIcon } from './InfoIcon' import { CloseIcon } from './CloseIcon' +import { ExplorerTxLink } from './ExplorerTxLink' const StyledErrorAlert = styled.div` border: 1px solid var(--failed-color); @@ -15,17 +16,33 @@ const CloseIconContainer = styled.div` ` const TextContainer = styled.div` + white-space: pre-wrap; flex-direction: column; ` -export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => ( -
- - - {error} - - - - -
-) +export const ErrorAlert = ({ onClick, error }: { onClick: () => void; error: string }) => { + const errorArray = error.split('%link') + const text = errorArray[0] + let link + if (errorArray.length > 1) { + link = ( + + {errorArray[1]} + + ) + } + return ( +
+ + + + {text} + {link} + + + + + +
+ ) +} diff --git a/alm/src/config/constants.ts b/alm/src/config/constants.ts index 5685440d3..8aaaf5d2f 100644 --- a/alm/src/config/constants.ts +++ b/alm/src/config/constants.ts @@ -18,9 +18,8 @@ export const ALM_HOME_TO_FOREIGN_MANUAL_EXECUTION: boolean = export const HOME_RPC_POLLING_INTERVAL: number = 5000 export const FOREIGN_RPC_POLLING_INTERVAL: number = 5000 -export const BLOCK_RANGE: number = 50 -export const ONE_DAY_TIMESTAMP: number = 86400 -export const THREE_DAYS_TIMESTAMP: number = 259200 +export const BLOCK_RANGE: number = 500 +export const MAX_TX_SEARCH_BLOCK_RANGE: number = 10000 export const EXECUTE_AFFIRMATION_HASH = 'e7a2c01f' export const SUBMIT_SIGNATURE_HASH = '630cea8e' @@ -66,3 +65,12 @@ export const VALIDATOR_CONFIRMATION_STATUS = { export const SEARCHING_TX = 'Searching Transaction...' export const INCORRECT_CHAIN_ERROR = `Incorrect chain chosen. Switch to ${FOREIGN_NETWORK_NAME} in the wallet.` + +export const DOUBLE_EXECUTION_ATTEMPT_ERROR = `Your execution transaction has been reverted. +However, the execution completed successfully in the transaction sent by a different party.` + +export const EXECUTION_FAILED_ERROR = `Your execution transaction has been reverted. +Please, contact the support by messaging on %linkhttps://forum.poa.network/c/support` + +export const EXECUTION_OUT_OF_GAS_ERROR = `Your execution transaction has been reverted due to Out-of-Gas error. +Please, resend the transaction and provide more gas to it.` diff --git a/alm/src/hooks/useClosestBlock.ts b/alm/src/hooks/useClosestBlock.ts new file mode 100644 index 000000000..67b08d9d4 --- /dev/null +++ b/alm/src/hooks/useClosestBlock.ts @@ -0,0 +1,68 @@ +import { useEffect, useState } from 'react' +import { TransactionReceipt } from 'web3-eth' +import { useStateProvider } from '../state/StateProvider' +import { FOREIGN_EXPLORER_API, HOME_EXPLORER_API } from '../config/constants' +import { getClosestBlockByTimestamp } from '../utils/explorer' + +export function useClosestBlock( + searchHome: boolean, + fromHome: boolean, + receipt: Maybe, + timestamp: number +) { + const { home, foreign } = useStateProvider() + const [blockNumber, setBlockNumber] = useState(null) + + useEffect( + () => { + if (!receipt || blockNumber || !timestamp) return + + if (fromHome === searchHome) { + setBlockNumber(receipt.blockNumber) + return + } + + const web3 = searchHome ? home.web3 : foreign.web3 + if (!web3) return + + const getBlock = async () => { + // try to fast-fetch closest block number from the chain explorer + try { + const api = searchHome ? HOME_EXPLORER_API : FOREIGN_EXPLORER_API + setBlockNumber(await getClosestBlockByTimestamp(api, timestamp)) + return + } catch {} + + const lastBlock = await web3.eth.getBlock('latest') + if (lastBlock.timestamp <= timestamp) { + setBlockNumber(lastBlock.number) + return + } + + const oldBlock = await web3.eth.getBlock(Math.max(lastBlock.number - 10000, 1)) + const blockDiff = lastBlock.number - oldBlock.number + const timeDiff = (lastBlock.timestamp as number) - (oldBlock.timestamp as number) + const averageBlockTime = timeDiff / blockDiff + let currentBlock = lastBlock + + let prevBlockDiff = Infinity + while (true) { + const timeDiff = (currentBlock.timestamp as number) - timestamp + const blockDiff = Math.ceil(timeDiff / averageBlockTime) + if (Math.abs(blockDiff) < 5 || Math.abs(blockDiff) >= Math.abs(prevBlockDiff)) { + setBlockNumber(currentBlock.number - blockDiff - 5) + break + } + + prevBlockDiff = blockDiff + currentBlock = await web3.eth.getBlock(currentBlock.number - blockDiff) + } + } + + getBlock() + }, + [blockNumber, foreign.web3, fromHome, home.web3, receipt, searchHome, timestamp] + ) + + return blockNumber +} diff --git a/alm/src/hooks/useMessageConfirmations.ts b/alm/src/hooks/useMessageConfirmations.ts index 5052cb22a..a7a204fca 100644 --- a/alm/src/hooks/useMessageConfirmations.ts +++ b/alm/src/hooks/useMessageConfirmations.ts @@ -28,7 +28,8 @@ export interface useMessageConfirmationsParams { message: MessageObject receipt: Maybe fromHome: boolean - timestamp: number + homeStartBlock: Maybe + foreignStartBlock: Maybe requiredSignatures: number validatorList: string[] blockConfirmations: number @@ -56,7 +57,8 @@ export const useMessageConfirmations = ({ message, receipt, fromHome, - timestamp, + homeStartBlock, + foreignStartBlock, requiredSignatures, validatorList, blockConfirmations @@ -90,6 +92,17 @@ export const useMessageConfirmations = ({ return filteredList.length > 0 } + // start watching blocks at the start + useEffect( + () => { + if (!home.web3 || !foreign.web3) return + + homeBlockNumberProvider.start(home.web3) + foreignBlockNumberProvider.start(foreign.web3) + }, + [foreign.web3, home.web3] + ) + // Check if the validators are waiting for block confirmations to verify the message useEffect( () => { @@ -105,9 +118,6 @@ export const useMessageConfirmations = ({ const blockProvider = fromHome ? homeBlockNumberProvider : foreignBlockNumberProvider const interval = fromHome ? HOME_RPC_POLLING_INTERVAL : FOREIGN_RPC_POLLING_INTERVAL - const web3 = fromHome ? home.web3 : foreign.web3 - blockProvider.start(web3) - const targetBlock = receipt.blockNumber + blockConfirmations checkSignaturesWaitingForBLocks( @@ -123,7 +133,6 @@ export const useMessageConfirmations = ({ return () => { unsubscribe() - blockProvider.stop() } }, [ @@ -153,8 +162,6 @@ export const useMessageConfirmations = ({ }) } - homeBlockNumberProvider.start(home.web3) - const fromBlock = receipt.blockNumber const toBlock = fromBlock + BLOCK_RANGE const messageHash = home.web3.utils.soliditySha3Raw(message.data) @@ -171,7 +178,6 @@ export const useMessageConfirmations = ({ return () => { unsubscribe() - homeBlockNumberProvider.stop() } }, [fromHome, home.bridgeContract, home.web3, message.data, receipt, signatureCollected] @@ -192,7 +198,6 @@ export const useMessageConfirmations = ({ }) } - homeBlockNumberProvider.start(home.web3) const targetBlock = collectedSignaturesEvent.blockNumber + blockConfirmations checkWaitingBlocksForExecution( @@ -208,7 +213,6 @@ export const useMessageConfirmations = ({ return () => { unsubscribe() - homeBlockNumberProvider.stop() } }, [collectedSignaturesEvent, fromHome, blockConfirmations, home.web3, receipt, waitingBlocksForExecutionResolved] @@ -218,7 +222,7 @@ export const useMessageConfirmations = ({ // To avoid making extra requests, this is only executed when validators finished waiting for blocks confirmations useEffect( () => { - if (!waitingBlocksResolved || !timestamp || !requiredSignatures) return + if (!waitingBlocksResolved || !homeStartBlock || !requiredSignatures) return const subscriptions: Array = [] @@ -239,7 +243,7 @@ export const useMessageConfirmations = ({ setSignatureCollected, waitingBlocksResolved, subscriptions, - timestamp, + homeStartBlock, getValidatorFailedTransactionsForMessage, setFailedConfirmations, getValidatorPendingTransactionsForMessage, @@ -259,7 +263,7 @@ export const useMessageConfirmations = ({ home.bridgeContract, requiredSignatures, waitingBlocksResolved, - timestamp, + homeStartBlock, setConfirmations ] ) @@ -270,6 +274,8 @@ export const useMessageConfirmations = ({ useEffect( () => { if ((fromHome && !waitingBlocksForExecutionResolved) || (!fromHome && !waitingBlocksResolved)) return + const startBlock = fromHome ? foreignStartBlock : homeStartBlock + if (!startBlock) return const subscriptions: Array = [] @@ -285,6 +291,7 @@ export const useMessageConfirmations = ({ const interval = fromHome ? FOREIGN_RPC_POLLING_INTERVAL : HOME_RPC_POLLING_INTERVAL getFinalizationEvent( + fromHome, bridgeContract, contractEvent, providedWeb3, @@ -293,7 +300,7 @@ export const useMessageConfirmations = ({ message, interval, subscriptions, - timestamp, + startBlock, collectedSignaturesEvent, getExecutionFailedTransactionForMessage, setFailedExecution, @@ -315,8 +322,9 @@ export const useMessageConfirmations = ({ home.web3, waitingBlocksResolved, waitingBlocksForExecutionResolved, - timestamp, - collectedSignaturesEvent + collectedSignaturesEvent, + foreignStartBlock, + homeStartBlock ] ) @@ -328,6 +336,9 @@ export const useMessageConfirmations = ({ ? CONFIRMATIONS_STATUS.SUCCESS : CONFIRMATIONS_STATUS.SUCCESS_MESSAGE_FAILED setStatus(newStatus) + + foreignBlockNumberProvider.stop() + homeBlockNumberProvider.stop() } else if (signatureCollected) { if (fromHome) { if (waitingBlocksForExecution) { diff --git a/alm/src/services/BlockNumberProvider.ts b/alm/src/services/BlockNumberProvider.ts index 686cf0038..41c401ce8 100644 --- a/alm/src/services/BlockNumberProvider.ts +++ b/alm/src/services/BlockNumberProvider.ts @@ -1,6 +1,6 @@ import Web3 from 'web3' import differenceInMilliseconds from 'date-fns/differenceInMilliseconds' -import { HOME_RPC_POLLING_INTERVAL } from '../config/constants' +import { FOREIGN_RPC_POLLING_INTERVAL, HOME_RPC_POLLING_INTERVAL } from '../config/constants' export class BlockNumberProvider { private running: number @@ -61,4 +61,4 @@ export class BlockNumberProvider { } export const homeBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL) -export const foreignBlockNumberProvider = new BlockNumberProvider(HOME_RPC_POLLING_INTERVAL) +export const foreignBlockNumberProvider = new BlockNumberProvider(FOREIGN_RPC_POLLING_INTERVAL) diff --git a/alm/src/utils/__tests__/explorer.test.ts b/alm/src/utils/__tests__/explorer.test.ts index d4743e6df..7cdf84172 100644 --- a/alm/src/utils/__tests__/explorer.test.ts +++ b/alm/src/utils/__tests__/explorer.test.ts @@ -17,19 +17,33 @@ const otherAddress = '0xD4075FB57fCf038bFc702c915Ef9592534bED5c1' describe('getFailedTransactions', () => { test('should only return failed transactions', async () => { - const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }] + const to = otherAddress + const transactions = [ + { isError: '0', to }, + { isError: '1', to }, + { isError: '0', to }, + { isError: '1', to }, + { isError: '1', to } + ] const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions) - const result = await getFailedTransactions('', '', 0, 1, '', fetchAccountTransactions) + const result = await getFailedTransactions('', to, 0, 1, '', fetchAccountTransactions) expect(result.length).toEqual(3) }) }) describe('getSuccessTransactions', () => { test('should only return success transactions', async () => { - const transactions = [{ isError: '0' }, { isError: '1' }, { isError: '0' }, { isError: '1' }, { isError: '1' }] + const to = otherAddress + const transactions = [ + { isError: '0', to }, + { isError: '1', to }, + { isError: '0', to }, + { isError: '1', to }, + { isError: '1', to } + ] const fetchAccountTransactions = jest.fn().mockImplementation(() => transactions) - const result = await getSuccessTransactions('', '', 0, 1, '', fetchAccountTransactions) + const result = await getSuccessTransactions('', to, 0, 1, '', fetchAccountTransactions) expect(result.length).toEqual(2) }) }) @@ -74,8 +88,8 @@ describe('getExecutionFailedTransactionForMessage', () => { account: '', to: '', messageData, - startTimestamp: 0, - endTimestamp: 1 + startBlock: 0, + endBlock: 1 }, fetchAccountTransactions ) diff --git a/alm/src/utils/__tests__/getFinalizationEvent.test.ts b/alm/src/utils/__tests__/getFinalizationEvent.test.ts index f6ce8ff1e..bbce1486b 100644 --- a/alm/src/utils/__tests__/getFinalizationEvent.test.ts +++ b/alm/src/utils/__tests__/getFinalizationEvent.test.ts @@ -64,6 +64,7 @@ describe('getFinalizationEvent', () => { const setExecutionEventsFetched = jest.fn() await getFinalizationEvent( + true, contract, eventName, web3, @@ -115,6 +116,7 @@ describe('getFinalizationEvent', () => { const setExecutionEventsFetched = jest.fn() await getFinalizationEvent( + true, contract, eventName, web3, @@ -166,6 +168,7 @@ describe('getFinalizationEvent', () => { const setExecutionEventsFetched = jest.fn() await getFinalizationEvent( + true, contract, eventName, web3, @@ -217,6 +220,7 @@ describe('getFinalizationEvent', () => { const setExecutionEventsFetched = jest.fn() await getFinalizationEvent( + true, contract, eventName, web3, @@ -275,6 +279,7 @@ describe('getFinalizationEvent', () => { const setExecutionEventsFetched = jest.fn() await getFinalizationEvent( + true, contract, eventName, web3, diff --git a/alm/src/utils/executionWaitingForBlocks.ts b/alm/src/utils/executionWaitingForBlocks.ts index 08ac77230..a221f3117 100644 --- a/alm/src/utils/executionWaitingForBlocks.ts +++ b/alm/src/utils/executionWaitingForBlocks.ts @@ -30,7 +30,6 @@ export const checkWaitingBlocksForExecution = async ( ) setWaitingBlocksForExecutionResolved(true) setWaitingBlocksForExecution(false) - blockProvider.stop() } else { let nextInterval = interval if (!currentBlock) { diff --git a/alm/src/utils/explorer.ts b/alm/src/utils/explorer.ts index 5b3548952..07293efc5 100644 --- a/alm/src/utils/explorer.ts +++ b/alm/src/utils/explorer.ts @@ -1,8 +1,10 @@ import { + BLOCK_RANGE, EXECUTE_AFFIRMATION_HASH, EXECUTE_SIGNATURES_HASH, FOREIGN_EXPLORER_API, HOME_EXPLORER_API, + MAX_TX_SEARCH_BLOCK_RANGE, SUBMIT_SIGNATURE_HASH } from '../config/constants' @@ -12,6 +14,7 @@ export interface APITransaction { input: string to: string hash: string + blockNumber: string } export interface APIPendingTransaction { @@ -27,144 +30,146 @@ export interface PendingTransactionsParams { export interface AccountTransactionsParams { account: string - to: string - startTimestamp: number - endTimestamp: number + startBlock: number + endBlock: number api: string } -export interface GetFailedTransactionParams { +export interface GetPendingTransactionParams { account: string to: string messageData: string - startTimestamp: number - endTimestamp: number } -export interface GetPendingTransactionParams { - account: string - to: string - messageData: string +export interface GetTransactionParams extends GetPendingTransactionParams { + startBlock: number + endBlock: number } -export const fetchAccountTransactionsFromBlockscout = async ({ - account, - to, - startTimestamp, - endTimestamp, - api -}: AccountTransactionsParams): Promise => { - const url = `${api}?module=account&action=txlist&address=${account}&filterby=from=${account}&to=${to}&starttimestamp=${startTimestamp}&endtimestamp=${endTimestamp}` +export const fetchAccountTransactions = async ({ account, startBlock, endBlock, api }: AccountTransactionsParams) => { + const params = `module=account&action=txlist&address=${account}&filterby=from&startblock=${startBlock}&endblock=${endBlock}` + const url = api.includes('blockscout') ? `${api}?${params}` : `${api}&${params}` - try { - const result = await fetch(url).then(res => res.json()) - if (result.status === '0') { - return [] - } + const result = await fetch(url).then(res => res.json()) - return result.result - } catch (e) { - console.log(e) + if (result.message === 'No transactions found') { return [] } -} -export const getBlockByTimestampUrl = (api: string, timestamp: number) => - `${api}&module=block&action=getblocknobytime×tamp=${timestamp}&closest=before` + return result.result +} -export const fetchAccountTransactionsFromEtherscan = async ({ +export const fetchPendingTransactions = async ({ account, - to, - startTimestamp, - endTimestamp, api -}: AccountTransactionsParams): Promise => { - const startBlockUrl = getBlockByTimestampUrl(api, startTimestamp) - const endBlockUrl = getBlockByTimestampUrl(api, endTimestamp) - let fromBlock = 0 - let toBlock = 9999999999999 - try { - const [fromBlockResult, toBlockResult] = await Promise.all([ - fetch(startBlockUrl).then(res => res.json()), - fetch(endBlockUrl).then(res => res.json()) - ]) - - if (fromBlockResult.status !== '0') { - fromBlock = parseInt(fromBlockResult.result) - } - - if (toBlockResult.status !== '0') { - toBlock = parseInt(toBlockResult.result) - } - } catch (e) { - console.log(e) +}: PendingTransactionsParams): Promise => { + if (!api.includes('blockscout')) { return [] } - - const url = `${api}&module=account&action=txlist&address=${account}&startblock=${fromBlock}&endblock=${toBlock}` + const url = `${api}?module=account&action=pendingtxlist&address=${account}` try { const result = await fetch(url).then(res => res.json()) - if (result.status === '0') { return [] } - const toAddressLowerCase = to.toLowerCase() - const transactions: APITransaction[] = result.result - return transactions.filter(t => t.to.toLowerCase() === toAddressLowerCase) + return result.result } catch (e) { - console.log(e) return [] } } -export const fetchAccountTransactions = (api: string) => { - return api.includes('blockscout') ? fetchAccountTransactionsFromBlockscout : fetchAccountTransactionsFromEtherscan +export const getClosestBlockByTimestamp = async (api: string, timestamp: number): Promise => { + if (api.includes('blockscout')) { + throw new Error('Blockscout does not support getblocknobytime') + } + + const url = `${api}&module=block&action=getblocknobytime×tamp=${timestamp}&closest=before` + + const blockNumber = await fetch(url).then(res => res.json()) + + return parseInt(blockNumber.result) } -export const fetchPendingTransactions = async ({ +// fast version of fetchAccountTransactions +// sequentially fetches transactions in small batches +// caches the result +const transactionsCache: { [key: string]: { lastBlock: number; transactions: APITransaction[] } } = {} +export const getAccountTransactions = async ({ account, + startBlock, + endBlock, api -}: PendingTransactionsParams): Promise => { - const url = `${api}?module=account&action=pendingtxlist&address=${account}` +}: AccountTransactionsParams): Promise => { + const key = `${account}-${startBlock}-${api}` - try { - const result = await fetch(url).then(res => res.json()) - if (result.status === '0') { - return [] + // initialize empty cache if it doesn't exist yet + if (!transactionsCache[key]) { + transactionsCache[key] = { lastBlock: startBlock - 1, transactions: [] } + } + + // if cache contains events up to block X, + // new batch is fetched for range [X + 1, X + 1 + BLOCK_RANGE] + const newStartBlock = transactionsCache[key].lastBlock + 1 + const newEndBlock = newStartBlock + BLOCK_RANGE + + // search for new transactions only if max allowed block range is not yet exceeded + if (newEndBlock <= startBlock + MAX_TX_SEARCH_BLOCK_RANGE) { + const newTransactions = await fetchAccountTransactions({ + account, + startBlock: newStartBlock, + endBlock: newEndBlock, + api + }) + + const transactions = transactionsCache[key].transactions.concat(...newTransactions) + + // cache updated transactions list + transactionsCache[key].transactions = transactions + + // enbBlock is assumed to be the current block number of the chain + // if the whole range is finalized, last block can be safely updated to the end of the range + // this works even if there are no transactions in the list + if (newEndBlock < endBlock) { + transactionsCache[key].lastBlock = newEndBlock + } else if (transactions.length > 0) { + transactionsCache[key].lastBlock = parseInt(transactions[transactions.length - 1].blockNumber, 10) } - return result.result - } catch (e) { - return [] + return transactions } + + console.warn(`Reached max transaction searching range, returning previously cached transactions for ${account}`) + return transactionsCache[key].transactions } +const filterReceiver = (to: string) => (tx: APITransaction) => tx.to.toLowerCase() === to.toLowerCase() + export const getFailedTransactions = async ( account: string, to: string, - startTimestamp: number, - endTimestamp: number, + startBlock: number, + endBlock: number, api: string, - fetchAccountTransactions: (args: AccountTransactionsParams) => Promise + getAccountTransactionsMethod = getAccountTransactions ): Promise => { - const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api }) + const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api }) - return transactions.filter(t => t.isError !== '0') + return transactions.filter(t => t.isError !== '0').filter(filterReceiver(to)) } export const getSuccessTransactions = async ( account: string, to: string, - startTimestamp: number, - endTimestamp: number, + startBlock: number, + endBlock: number, api: string, - fetchAccountTransactions: (args: AccountTransactionsParams) => Promise + getAccountTransactionsMethod = getAccountTransactions ): Promise => { - const transactions = await fetchAccountTransactions({ account, to, startTimestamp, endTimestamp, api }) + const transactions = await getAccountTransactionsMethod({ account, startBlock, endBlock, api }) - return transactions.filter(t => t.isError === '0') + return transactions.filter(t => t.isError === '0').filter(filterReceiver(to)) } export const filterValidatorSignatureTransaction = ( @@ -183,17 +188,10 @@ export const getValidatorFailedTransactionsForMessage = async ({ account, to, messageData, - startTimestamp, - endTimestamp -}: GetFailedTransactionParams): Promise => { - const failedTransactions = await getFailedTransactions( - account, - to, - startTimestamp, - endTimestamp, - HOME_EXPLORER_API, - fetchAccountTransactionsFromBlockscout - ) + startBlock, + endBlock +}: GetTransactionParams): Promise => { + const failedTransactions = await getFailedTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API) return filterValidatorSignatureTransaction(failedTransactions, messageData) } @@ -202,33 +200,19 @@ export const getValidatorSuccessTransactionsForMessage = async ({ account, to, messageData, - startTimestamp, - endTimestamp -}: GetFailedTransactionParams): Promise => { - const transactions = await getSuccessTransactions( - account, - to, - startTimestamp, - endTimestamp, - HOME_EXPLORER_API, - fetchAccountTransactionsFromBlockscout - ) + startBlock, + endBlock +}: GetTransactionParams): Promise => { + const transactions = await getSuccessTransactions(account, to, startBlock, endBlock, HOME_EXPLORER_API) return filterValidatorSignatureTransaction(transactions, messageData) } export const getExecutionFailedTransactionForMessage = async ( - { account, to, messageData, startTimestamp, endTimestamp }: GetFailedTransactionParams, + { account, to, messageData, startBlock, endBlock }: GetTransactionParams, getFailedTransactionsMethod = getFailedTransactions ): Promise => { - const failedTransactions = await getFailedTransactionsMethod( - account, - to, - startTimestamp, - endTimestamp, - FOREIGN_EXPLORER_API, - fetchAccountTransactions(FOREIGN_EXPLORER_API) - ) + const failedTransactions = await getFailedTransactionsMethod(account, to, startBlock, endBlock, FOREIGN_EXPLORER_API) const messageDataValue = messageData.replace('0x', '') return failedTransactions.filter(t => t.input.includes(EXECUTE_SIGNATURES_HASH) && t.input.includes(messageDataValue)) diff --git a/alm/src/utils/getCollectedSignaturesEvent.ts b/alm/src/utils/getCollectedSignaturesEvent.ts index 312c1a5f8..003a60954 100644 --- a/alm/src/utils/getCollectedSignaturesEvent.ts +++ b/alm/src/utils/getCollectedSignaturesEvent.ts @@ -31,7 +31,6 @@ export const getCollectedSignaturesEvent = async ( if (filteredEvents.length) { const event = filteredEvents[0] setCollectedSignaturesEvent(event) - homeBlockNumberProvider.stop() } else { const newFromBlock = currentBlock ? securedToBlock : fromBlock const newToBlock = currentBlock ? toBlock + BLOCK_RANGE : toBlock diff --git a/alm/src/utils/getConfirmationsForTx.ts b/alm/src/utils/getConfirmationsForTx.ts index c8ab7c67e..015793d0f 100644 --- a/alm/src/utils/getConfirmationsForTx.ts +++ b/alm/src/utils/getConfirmationsForTx.ts @@ -1,12 +1,7 @@ import Web3 from 'web3' import { Contract } from 'web3-eth-contract' import { HOME_RPC_POLLING_INTERVAL, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' -import { - GetFailedTransactionParams, - APITransaction, - APIPendingTransaction, - GetPendingTransactionParams -} from './explorer' +import { GetTransactionParams, APITransaction, APIPendingTransaction, GetPendingTransactionParams } from './explorer' import { getAffirmationsSigned, getMessagesSigned } from './contract' import { getValidatorConfirmation, @@ -43,12 +38,12 @@ export const getConfirmationsForTx = async ( setSignatureCollected: Function, waitingBlocksResolved: boolean, subscriptions: number[], - timestamp: number, - getFailedTransactions: (args: GetFailedTransactionParams) => Promise, + startBlock: number, + getFailedTransactions: (args: GetTransactionParams) => Promise, setFailedConfirmations: Function, getPendingTransactions: (args: GetPendingTransactionParams) => Promise, setPendingConfirmations: Function, - getSuccessTransactions: (args: GetFailedTransactionParams) => Promise + getSuccessTransactions: (args: GetTransactionParams) => Promise ) => { if (!web3 || !validatorList || !validatorList.length || !bridgeContract || !waitingBlocksResolved) return @@ -102,7 +97,7 @@ export const getConfirmationsForTx = async ( // Check if confirmation failed const validatorFailedConfirmationsChecks = await Promise.all( undefinedConfirmations.map( - getValidatorFailedTransaction(bridgeContract, messageData, timestamp, getFailedTransactions) + getValidatorFailedTransaction(bridgeContract, messageData, startBlock, getFailedTransactions) ) ) const validatorFailedConfirmations = validatorFailedConfirmationsChecks.filter( @@ -136,7 +131,7 @@ export const getConfirmationsForTx = async ( // get transactions from success signatures const successConfirmationWithData = await Promise.all( successConfirmations.map( - getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, timestamp, getSuccessTransactions) + getSuccessExecutionTransaction(web3, bridgeContract, fromHome, messageData, startBlock, getSuccessTransactions) ) ) @@ -162,7 +157,7 @@ export const getConfirmationsForTx = async ( setSignatureCollected, waitingBlocksResolved, subscriptions, - timestamp, + startBlock, getFailedTransactions, setFailedConfirmations, getPendingTransactions, diff --git a/alm/src/utils/getFinalizationEvent.ts b/alm/src/utils/getFinalizationEvent.ts index 8f383a087..3406007e9 100644 --- a/alm/src/utils/getFinalizationEvent.ts +++ b/alm/src/utils/getFinalizationEvent.ts @@ -1,41 +1,20 @@ import { Contract, EventData } from 'web3-eth-contract' import Web3 from 'web3' -import { CACHE_KEY_EXECUTION_FAILED, THREE_DAYS_TIMESTAMP, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' +import { CACHE_KEY_EXECUTION_FAILED, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' import { ExecutionData } from '../hooks/useMessageConfirmations' -import { - APIPendingTransaction, - APITransaction, - GetFailedTransactionParams, - GetPendingTransactionParams -} from './explorer' +import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer' import { getBlock, MessageObject } from './web3' import validatorsCache from '../services/ValidatorsCache' +import { foreignBlockNumberProvider, homeBlockNumberProvider } from '../services/BlockNumberProvider' -export const getFinalizationEvent = async ( - contract: Maybe, - eventName: string, - web3: Maybe, - setResult: React.Dispatch>, - waitingBlocksResolved: boolean, - message: MessageObject, - interval: number, - subscriptions: number[], - timestamp: number, - collectedSignaturesEvent: Maybe, - getFailedExecution: (args: GetFailedTransactionParams) => Promise, - setFailedExecution: Function, - getPendingExecution: (args: GetPendingTransactionParams) => Promise, - setPendingExecution: Function, - setExecutionEventsFetched: Function -) => { - if (!contract || !web3 || !waitingBlocksResolved) return +export const getSuccessExecutionData = async (contract: Contract, eventName: string, web3: Web3, messageId: string) => { // Since it filters by the message id, only one event will be fetched // so there is no need to limit the range of the block to reduce the network traffic const events: EventData[] = await contract.getPastEvents(eventName, { fromBlock: 0, toBlock: 'latest', filter: { - messageId: message.id + messageId } }) if (events.length > 0) { @@ -48,13 +27,39 @@ export const getFinalizationEvent = async ( const blockTimestamp = typeof block.timestamp === 'string' ? parseInt(block.timestamp) : block.timestamp const validatorAddress = web3.utils.toChecksumAddress(txReceipt.from) - setResult({ + return { status: VALIDATOR_CONFIRMATION_STATUS.SUCCESS, validator: validatorAddress, txHash: event.transactionHash, timestamp: blockTimestamp, executionResult: event.returnValues.status - }) + } + } + return null +} + +export const getFinalizationEvent = async ( + fromHome: boolean, + contract: Maybe, + eventName: string, + web3: Maybe, + setResult: React.Dispatch>, + waitingBlocksResolved: boolean, + message: MessageObject, + interval: number, + subscriptions: number[], + startBlock: number, + collectedSignaturesEvent: Maybe, + getFailedExecution: (args: GetTransactionParams) => Promise, + setFailedExecution: Function, + getPendingExecution: (args: GetPendingTransactionParams) => Promise, + setPendingExecution: Function, + setExecutionEventsFetched: Function +) => { + if (!contract || !web3 || !waitingBlocksResolved) return + const successExecutionData = await getSuccessExecutionData(contract, eventName, web3, message.id) + if (successExecutionData) { + setResult(successExecutionData) } else { setExecutionEventsFetched(true) // If event is defined, it means it is a message from Home to Foreign @@ -84,14 +89,15 @@ export const getFinalizationEvent = async ( } else { const validatorExecutionCacheKey = `${CACHE_KEY_EXECUTION_FAILED}${validator}-${message.id}` const failedFromCache = validatorsCache.get(validatorExecutionCacheKey) + const blockProvider = fromHome ? foreignBlockNumberProvider : homeBlockNumberProvider if (!failedFromCache) { const failedTransactions = await getFailedExecution({ account: validator, to: contract.options.address, messageData: message.data, - startTimestamp: timestamp, - endTimestamp: timestamp + THREE_DAYS_TIMESTAMP + startBlock, + endBlock: blockProvider.get() || 0 }) if (failedTransactions.length > 0) { @@ -117,6 +123,7 @@ export const getFinalizationEvent = async ( const timeoutId = setTimeout( () => getFinalizationEvent( + fromHome, contract, eventName, web3, @@ -125,7 +132,7 @@ export const getFinalizationEvent = async ( message, interval, subscriptions, - timestamp, + startBlock, collectedSignaturesEvent, getFailedExecution, setFailedExecution, diff --git a/alm/src/utils/signatureWaitingForBlocks.ts b/alm/src/utils/signatureWaitingForBlocks.ts index 9133d364c..51db374df 100644 --- a/alm/src/utils/signatureWaitingForBlocks.ts +++ b/alm/src/utils/signatureWaitingForBlocks.ts @@ -16,7 +16,6 @@ export const checkSignaturesWaitingForBLocks = async ( if (currentBlock && currentBlock >= targetBlock) { setWaitingBlocksResolved(true) setWaitingStatus(false) - blockProvider.stop() } else { let nextInterval = interval if (!currentBlock) { diff --git a/alm/src/utils/validatorConfirmationHelpers.ts b/alm/src/utils/validatorConfirmationHelpers.ts index 4ac872c76..24f364a9b 100644 --- a/alm/src/utils/validatorConfirmationHelpers.ts +++ b/alm/src/utils/validatorConfirmationHelpers.ts @@ -2,18 +2,9 @@ import Web3 from 'web3' import { Contract } from 'web3-eth-contract' import { BasicConfirmationParam, ConfirmationParam } from '../hooks/useMessageConfirmations' import validatorsCache from '../services/ValidatorsCache' -import { - CACHE_KEY_FAILED, - CACHE_KEY_SUCCESS, - ONE_DAY_TIMESTAMP, - VALIDATOR_CONFIRMATION_STATUS -} from '../config/constants' -import { - APIPendingTransaction, - APITransaction, - GetFailedTransactionParams, - GetPendingTransactionParams -} from './explorer' +import { CACHE_KEY_FAILED, CACHE_KEY_SUCCESS, VALIDATOR_CONFIRMATION_STATUS } from '../config/constants' +import { APIPendingTransaction, APITransaction, GetTransactionParams, GetPendingTransactionParams } from './explorer' +import { homeBlockNumberProvider } from '../services/BlockNumberProvider' export const getValidatorConfirmation = ( web3: Web3, @@ -50,8 +41,8 @@ export const getSuccessExecutionTransaction = ( bridgeContract: Contract, fromHome: boolean, messageData: string, - timestamp: number, - getSuccessTransactions: (args: GetFailedTransactionParams) => Promise + startBlock: number, + getSuccessTransactions: (args: GetTransactionParams) => Promise ) => async (validatorData: BasicConfirmationParam): Promise => { const { validator } = validatorData const validatorCacheKey = `${CACHE_KEY_SUCCESS}${validatorData.validator}-${messageData}` @@ -65,8 +56,8 @@ export const getSuccessExecutionTransaction = ( account: validatorData.validator, to: bridgeContract.options.address, messageData, - startTimestamp: timestamp, - endTimestamp: timestamp + ONE_DAY_TIMESTAMP + startBlock, + endBlock: homeBlockNumberProvider.get() || 0 }) let txHashTimestamp = 0 @@ -98,8 +89,8 @@ export const getSuccessExecutionTransaction = ( export const getValidatorFailedTransaction = ( bridgeContract: Contract, messageData: string, - timestamp: number, - getFailedTransactions: (args: GetFailedTransactionParams) => Promise + startBlock: number, + getFailedTransactions: (args: GetTransactionParams) => Promise ) => async (validatorData: BasicConfirmationParam): Promise => { const validatorCacheKey = `${CACHE_KEY_FAILED}${validatorData.validator}-${messageData}` const failedFromCache = validatorsCache.getData(validatorCacheKey) @@ -112,8 +103,8 @@ export const getValidatorFailedTransaction = ( account: validatorData.validator, to: bridgeContract.options.address, messageData, - startTimestamp: timestamp, - endTimestamp: timestamp + ONE_DAY_TIMESTAMP + startBlock, + endBlock: homeBlockNumberProvider.get() || 0 }) const newStatus = failedTransactions.length > 0 ? VALIDATOR_CONFIRMATION_STATUS.FAILED : VALIDATOR_CONFIRMATION_STATUS.UNDEFINED diff --git a/deployment/roles/common/tasks/dependencies.yml b/deployment/roles/common/tasks/dependencies.yml index 4bc82c503..0c859c3d8 100644 --- a/deployment/roles/common/tasks/dependencies.yml +++ b/deployment/roles/common/tasks/dependencies.yml @@ -33,7 +33,7 @@ mode: "0755" - name: Upgrade pip version - shell: pip3 install --upgrade pip + shell: pip3 install --upgrade pip==19.3.1 - name: Install python docker library shell: pip3 install docker docker-compose setuptools diff --git a/monitor/checkWorker3.js b/monitor/checkWorker3.js index 08ad3dad2..c7c271604 100644 --- a/monitor/checkWorker3.js +++ b/monitor/checkWorker3.js @@ -2,6 +2,7 @@ require('dotenv').config() const logger = require('./logger')('checkWorker3') const stuckTransfers = require('./stuckTransfers') const detectMediators = require('./detectMediators') +const detectFailures = require('./detectFailures') const { writeFile, createDir } = require('./utils/file') const { web3Home } = require('./utils/web3') const { saveCache } = require('./utils/web3Cache') @@ -24,11 +25,19 @@ async function checkWorker3() { logger.debug('Done') } else if (bridgeMode === BRIDGE_MODES.ARBITRARY_MESSAGE) { createDir(`/responses/${MONITOR_BRIDGE_NAME}`) + logger.debug('calling detectMediators()') const mediators = await detectMediators(bridgeMode) mediators.ok = true mediators.health = true writeFile(`/responses/${MONITOR_BRIDGE_NAME}/mediators.json`, mediators) + + logger.debug('calling detectFailures()') + const failures = await detectFailures(bridgeMode) + failures.ok = true + failures.health = true + writeFile(`/responses/${MONITOR_BRIDGE_NAME}/failures.json`, failures) + saveCache() logger.debug('Done') } diff --git a/monitor/detectFailures.js b/monitor/detectFailures.js new file mode 100644 index 000000000..81d97b4db --- /dev/null +++ b/monitor/detectFailures.js @@ -0,0 +1,86 @@ +require('dotenv').config() +const logger = require('./logger')('alerts') +const eventsInfo = require('./utils/events') +const { normalizeAMBMessageEvent } = require('../commons') +const { getHomeBlockNumber, getForeignBlockNumber } = require('./utils/web3') + +function normalize(events) { + const requests = {} + events.forEach(event => { + const request = normalizeAMBMessageEvent(event) + request.requestTx = event.transactionHash + requests[request.messageId] = request + }) + return confirmation => { + const request = requests[confirmation.returnValues.messageId] || {} + return { + ...request, + status: false, + executionTx: confirmation.transactionHash, + executionBlockNumber: confirmation.blockNumber + } + } +} + +async function main(mode) { + const { + homeToForeignRequests, + homeToForeignConfirmations, + foreignToHomeConfirmations, + foreignToHomeRequests + } = await eventsInfo(mode) + const hasFailed = event => !event.returnValues.status + const cmp = (a, b) => b.executionBlockNumber - a.executionBlockNumber + const failedForeignToHomeMessages = foreignToHomeConfirmations + .filter(hasFailed) + .map(normalize(foreignToHomeRequests)) + .sort(cmp) + const failedHomeToForeignMessages = homeToForeignConfirmations + .filter(hasFailed) + .map(normalize(homeToForeignRequests)) + .sort(cmp) + + const homeBlockNumber = await getHomeBlockNumber() + const foreignBlockNumber = await getForeignBlockNumber() + + const blockRanges = [1000, 10000, 100000, 1000000] + const rangeNames = [ + `last${blockRanges[0]}blocks`, + ...blockRanges.slice(0, blockRanges.length - 1).map((n, i) => `last${n}to${blockRanges[i + 1]}blocks`), + `before${blockRanges[blockRanges.length - 1]}blocks` + ] + + const countFailures = (failedMessages, lastBlockNumber) => { + const result = {} + rangeNames.forEach(name => { + result[name] = 0 + }) + failedMessages.forEach(message => { + const blockAge = lastBlockNumber - message.executionBlockNumber + let rangeIndex = blockRanges.findIndex(n => n > blockAge) + if (rangeIndex === -1) { + rangeIndex = blockRanges.length + } + result[rangeNames[rangeIndex]] += 1 + }) + return result + } + + logger.debug('Done') + + return { + homeToForeign: { + total: failedHomeToForeignMessages.length, + stats: countFailures(failedHomeToForeignMessages, foreignBlockNumber), + lastFailures: failedHomeToForeignMessages.slice(0, 5) + }, + foreignToHome: { + total: failedForeignToHomeMessages.length, + stats: countFailures(failedForeignToHomeMessages, homeBlockNumber), + lastFailures: failedForeignToHomeMessages.slice(0, 5) + }, + lastChecked: Math.floor(Date.now() / 1000) + } +} + +module.exports = main diff --git a/monitor/index.js b/monitor/index.js index 3aaaa8200..c2f3d6f30 100644 --- a/monitor/index.js +++ b/monitor/index.js @@ -2,6 +2,7 @@ require('dotenv').config() const express = require('express') const cors = require('cors') const { readFile } = require('./utils/file') +const { getPrometheusMetrics } = require('./prometheusMetrics') const app = express() const bridgeRouter = express.Router({ mergeParams: true }) @@ -11,9 +12,10 @@ app.use(cors()) app.get('/favicon.ico', (req, res) => res.sendStatus(204)) app.use('/:bridgeName', bridgeRouter) -bridgeRouter.get('/', async (req, res, next) => { +bridgeRouter.get('/:file(validators|eventsStats|alerts|mediators|stuckTransfers|failures)?', (req, res, next) => { try { - const results = await readFile(`./responses/${req.params.bridgeName}/getBalances.json`) + const { bridgeName, file } = req.params + const results = readFile(`./responses/${bridgeName}/${file || 'getBalances'}.json`) res.json(results) } catch (e) { // this will eventually be handled by your error handling middleware @@ -21,49 +23,10 @@ bridgeRouter.get('/', async (req, res, next) => { } }) -bridgeRouter.get('/validators', async (req, res, next) => { +bridgeRouter.get('/metrics', (req, res, next) => { try { - const results = await readFile(`./responses/${req.params.bridgeName}/validators.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/eventsStats', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/eventsStats.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/alerts', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/alerts.json`) - res.json(results) - } catch (e) { - next(e) - } -}) - -bridgeRouter.get('/mediators', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/mediators.json`) - res.json(results) - } catch (e) { - // this will eventually be handled by your error handling middleware - next(e) - } -}) - -bridgeRouter.get('/stuckTransfers', async (req, res, next) => { - try { - const results = await readFile(`./responses/${req.params.bridgeName}/stuckTransfers.json`) - res.json(results) + const metrics = getPrometheusMetrics(req.params.bridgeName) + res.type('text').send(metrics) } catch (e) { next(e) } diff --git a/monitor/prometheusMetrics.js b/monitor/prometheusMetrics.js new file mode 100644 index 000000000..af649abce --- /dev/null +++ b/monitor/prometheusMetrics.js @@ -0,0 +1,104 @@ +const { readFile } = require('./utils/file') + +const { + MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST, + MONITOR_HOME_TO_FOREIGN_BLOCK_LIST, + MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, + MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE +} = process.env + +function BridgeConf(type, validatorsBalanceEnable, alertTargetFunc, failureDirection) { + this.type = type + this.validatorsBalanceEnable = validatorsBalanceEnable + this.alertTargetFunc = alertTargetFunc + this.failureDirection = failureDirection +} + +const BRIDGE_CONFS = [ + new BridgeConf('home', MONITOR_HOME_VALIDATORS_BALANCE_ENABLE, 'executeAffirmations', 'homeToForeign'), + new BridgeConf('foreign', MONITOR_FOREIGN_VALIDATORS_BALANCE_ENABLE, 'executeSignatures', 'foreignToHome') +] + +function hasError(obj) { + return 'error' in obj +} + +function getPrometheusMetrics(bridgeName) { + const responsePath = jsonName => `./responses/${bridgeName}/${jsonName}.json` + + const metrics = {} + + // Balance metrics + const balancesFile = readFile(responsePath('getBalances')) + + if (!hasError(balancesFile)) { + const { home: homeBalances, foreign: foreignBalances, ...commonBalances } = balancesFile + metrics.balances_home_value = homeBalances.totalSupply + metrics.balances_home_txs_deposit = homeBalances.deposits + metrics.balances_home_txs_withdrawal = homeBalances.withdrawals + + metrics.balances_foreign_value = foreignBalances.erc20Balance + metrics.balances_foreign_txs_deposit = foreignBalances.deposits + metrics.balances_foreign_txs_withdrawal = foreignBalances.withdrawals + + metrics.balances_diff_value = commonBalances.balanceDiff + metrics.balances_diff_deposit = commonBalances.depositsDiff + metrics.balances_diff_withdrawal = commonBalances.withdrawalDiff + if (MONITOR_HOME_TO_FOREIGN_ALLOWANCE_LIST || MONITOR_HOME_TO_FOREIGN_BLOCK_LIST) { + metrics.balances_unclaimed_txs = commonBalances.unclaimedDiff + metrics.balances_unclaimed_value = commonBalances.unclaimedBalance + } + } + + // Validator metrics + const validatorsFile = readFile(responsePath('validators')) + + if (!hasError(validatorsFile)) { + for (const bridge of BRIDGE_CONFS) { + const allValidators = validatorsFile[bridge.type].validators + const validatorAddressesWithBalanceCheck = + typeof bridge.validatorsBalanceEnable === 'string' + ? bridge.validatorsBalanceEnable.split(' ') + : Object.keys(allValidators) + + validatorAddressesWithBalanceCheck.forEach((addr, ind) => { + metrics[`validators_balances_${bridge.type}${ind}{address="${addr}"}`] = allValidators[addr].balance + }) + } + } + + // Alert metrics + const alertsFile = readFile(responsePath('alerts')) + + if (!hasError(alertsFile)) { + for (const bridge of BRIDGE_CONFS) { + Object.entries(alertsFile[bridge.alertTargetFunc].misbehavior).forEach(([period, val]) => { + metrics[`misbehavior_${bridge.type}_${period}`] = val + }) + } + } + + // Failure metrics + const failureFile = readFile(responsePath('failures')) + + if (!hasError(failureFile)) { + for (const bridge of BRIDGE_CONFS) { + const dir = bridge.failureDirection + const failures = failureFile[dir] + metrics[`failures_${dir}_total`] = failures.total + Object.entries(failures.stats).forEach(([period, count]) => { + metrics[`failures_${dir}_${period}`] = count + }) + } + } + + // Pack metrcis into a plain text + return Object.entries(metrics).reduceRight( + // Prometheus supports `Nan` and possibly signed `Infinity` + // in case cast to `Number` fails + (acc, [key, val]) => `${key} ${val ? Number(val) : 0}\n${acc}`, + '' + ) +} + +module.exports = { getPrometheusMetrics } diff --git a/monitor/utils/file.js b/monitor/utils/file.js index ad9f25f61..fc957fa87 100644 --- a/monitor/utils/file.js +++ b/monitor/utils/file.js @@ -1,9 +1,9 @@ const fs = require('fs') const path = require('path') -async function readFile(filePath) { +function readFile(filePath) { try { - const content = await fs.readFileSync(filePath) + const content = fs.readFileSync(filePath) const json = JSON.parse(content) const timeDiff = Math.floor(Date.now() / 1000) - json.lastChecked return Object.assign({}, json, { timeDiff }) diff --git a/oracle/src/sender.js b/oracle/src/sender.js index 172631a54..659d0b5fc 100644 --- a/oracle/src/sender.js +++ b/oracle/src/sender.js @@ -173,15 +173,18 @@ async function main({ msg, ackMsg, nackMsg, channel, scheduleForRetry, scheduleT `Tx Failed for event Tx ${job.transactionReference}.`, e.message ) - if (!e.message.toLowerCase().includes('transaction with the same hash was already imported')) { - if (isResend) { - resendJobs.push(job) - } else { - failedTx.push(job) - } + + const message = e.message.toLowerCase() + if (isResend || message.includes('transaction with the same hash was already imported')) { + resendJobs.push(job) + } else { + // if initial transaction sending has failed not due to the same hash error + // send it to the failed tx queue + // this will result in the sooner resend attempt than if using resendJobs + failedTx.push(job) } - if (e.message.toLowerCase().includes('insufficient funds')) { + if (message.includes('insufficient funds')) { insufficientFunds = true const currentBalance = await web3Instance.eth.getBalance(ORACLE_VALIDATOR_ADDRESS) minimumBalance = gasLimit.multipliedBy(gasPrice)