From 2ee51cdf57b2d1081c470338bc83235c4990c732 Mon Sep 17 00:00:00 2001 From: Gustavo Inacio Date: Wed, 28 Aug 2024 19:20:34 +0200 Subject: [PATCH] fix: use subgraph timestamp for tap Signed-off-by: Gustavo Inacio --- .../src/allocations/query-fees.ts | 145 +++++++++++++----- 1 file changed, 109 insertions(+), 36 deletions(-) diff --git a/packages/indexer-common/src/allocations/query-fees.ts b/packages/indexer-common/src/allocations/query-fees.ts index 9d9434671..325d09fe2 100644 --- a/packages/indexer-common/src/allocations/query-fees.ts +++ b/packages/indexer-common/src/allocations/query-fees.ts @@ -106,6 +106,21 @@ interface RavWithAllocation { sender: Address } +interface TapSubgraphResponse { + transactions: { + allocationID: string + timestamp: number + sender: { + id: string + } + }[] + _meta: { + block: { + timestamp: number + } + } +} + export class AllocationReceiptCollector implements ReceiptCollector { declare logger: Logger declare metrics: ReceiptMetrics @@ -564,73 +579,121 @@ export class AllocationReceiptCollector implements ReceiptCollector { // redeem only if last is true // Later can add order and limit private async pendingRAVs(): Promise { - const unfinalizedRAVs = await this.models.receiptAggregateVouchers.findAll({ + const ravLastNotFinal = await this.models.receiptAggregateVouchers.findAll({ where: { last: true, final: false }, }) // Obtain allocationIds to use as filter in subgraph - const unfinalizedRavsAllocationIds = unfinalizedRAVs.map((rav) => + const ravLastNotFinalAllocationIds = ravLastNotFinal.map((rav) => [ rav.getSignedRAV().rav.allocationId.toLowerCase(), - ) + rav.senderAddress, + ]) - if (unfinalizedRavsAllocationIds.length > 0) { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - let tapSubgraphResponse: any + if (ravLastNotFinalAllocationIds.length > 0) { + let tapSubgraphResponse: { data?: TapSubgraphResponse } if (!this.tapSubgraph) { - tapSubgraphResponse = { data: { transactions: [] } } + tapSubgraphResponse = { + data: { transactions: [], _meta: { block: { timestamp: 0 } } }, + } } else { - tapSubgraphResponse = await this.tapSubgraph!.query( + tapSubgraphResponse = await this.tapSubgraph!.query( gql` query transactions($unfinalizedRavsAllocationIds: [String!]!) { transactions( where: { type: "redeem", allocationID_in: $unfinalizedRavsAllocationIds } ) { allocationID + timestamp + sender { + id + } + } + _meta { + block { + timestamp + } } } `, - { unfinalizedRavsAllocationIds }, + { unfinalizedRavsAllocationIds: ravLastNotFinalAllocationIds }, ) } - const alreadyRedeemedAllocations = tapSubgraphResponse.data.transactions.map( - (transaction) => transaction.allocationID, + + const alreadyRedeemedAllocations = tapSubgraphResponse.data!.transactions.map( + (transaction) => [transaction.allocationID, transaction.sender.id], ) + const redeemedRavsNotOnOurDatabase = tapSubgraphResponse.data!.transactions.filter( + (data) => { + ravLastNotFinalAllocationIds.includes([data.allocationID, data.sender.id]) + }, + ) + + // for each transaction that is not redeemed on our database + // but was redeemed on the blockchain, update it to redeemed + if (redeemedRavsNotOnOurDatabase.length > 0) { + for (let rav of redeemedRavsNotOnOurDatabase) { + await this.markRavAsRedeemed(rav.allocationID, rav.sender.id, rav.timestamp) + } + } + // Filter unfinalized RAVS fetched from DB, keeping RAVs that have not yet been redeemed on-chain - const nonRedeemedAllocationIDAddresses = unfinalizedRavsAllocationIds.filter( + const nonRedeemedAllocationIDAddresses = ravLastNotFinalAllocationIds.filter( (allocationID) => !alreadyRedeemedAllocations.includes(allocationID), ) // Lowercase and remove '0x' prefix of addresses to match format in TAP DB Tables const nonRedeemedAllocationIDsTrunc = nonRedeemedAllocationIDAddresses.map( - (allocationID) => allocationID.toLowerCase().replace('0x', ''), + (allocationID) => allocationID[0].toLowerCase().replace('0x', ''), ) + // we use the subgraph timestamp to make decisions + // block timestamp minus 1 minute (because of blockchain timestamp uncertainty) + const ONE_MINUTE = 60 + const blockTimestampSecs = + tapSubgraphResponse.data!._meta.block.timestamp - ONE_MINUTE + // Mark RAVs as unredeemed in DB if the TAP subgraph couldn't find the redeem Tx. // To handle a chain reorg that "unredeemed" the RAVs. // WE use sql directly due to a bug in sequelize update: // https://github.com/sequelize/sequelize/issues/7664 (bug been open for 7 years no fix yet or ever) + if (nonRedeemedAllocationIDsTrunc.length > 0) { + await this.revertRavsRedeemed(nonRedeemedAllocationIDsTrunc, blockTimestampSecs) + } + + // For all RAVs that passed finality time, we mark it as final + await this.markRavsAsFinal(blockTimestampSecs) + + return await this.models.receiptAggregateVouchers.findAll({ + where: { redeemedAt: null, final: false, last: true }, + }) + } + return [] + } - let query = ` + // for every allocation_id of this list that contains the timestamp_ns less than the current + // subgraph timestamp + private async revertRavsRedeemed(allocation_ids: string[], blockTimestampSecs: number) { + const SECONDS_TO_NANOSECONDS = 1000000000 + const blockTimestampNs = blockTimestampSecs * SECONDS_TO_NANOSECONDS + let query = ` UPDATE scalar_tap_ravs SET redeemed_at = NULL - WHERE allocation_id IN ('${nonRedeemedAllocationIDsTrunc.join("', '")}') + WHERE allocation_id IN ('${allocation_ids.join("', '")}') + AND timetstamp_ns < ${blockTimestampNs} ` - await this.models.receiptAggregateVouchers.sequelize?.query(query) + await this.models.receiptAggregateVouchers.sequelize?.query(query) + } - // // Update those that redeemed_at is older than 60 minutes and mark as final - query = ` + // we use blockTimestamp instead of NOW() because we must be older than + // the subgraph timestamp + private async markRavsAsFinal(blockTimestampSecs: number) { + const query = ` UPDATE scalar_tap_ravs SET final = TRUE WHERE last = TRUE AND final = FALSE - AND redeemed_at < NOW() - INTERVAL '${this.finalityTime} second' + AND redeemed_at < ${blockTimestampSecs - this.finalityTime} AND redeemed_at IS NOT NULL ` - await this.models.receiptAggregateVouchers.sequelize?.query(query) - - return await this.models.receiptAggregateVouchers.findAll({ - where: { redeemedAt: null, final: false, last: true }, - }) - } - return [] + await this.models.receiptAggregateVouchers.sequelize?.query(query) } private encodeReceiptBatch(receipts: AllocationReceipt[]): BytesWriter { @@ -942,18 +1005,12 @@ export class AllocationReceiptCollector implements ReceiptCollector { ) try { - const addressWithoutPrefix = rav.allocationId.toLowerCase().replace('0x', '') - // WE use sql directly due to a bug in sequelize update: - // https://github.com/sequelize/sequelize/issues/7664 (bug been open for 7 years no fix yet or ever) - const query = ` - UPDATE scalar_tap_ravs - SET redeemed_at = NOW() - WHERE allocation_id = '${addressWithoutPrefix}' - ` - await this.models.receiptAggregateVouchers.sequelize?.query(query) + const allocationId = rav.allocationId.toLowerCase().replace('0x', '') + const senderAddress = sender.toLowerCase().replace('0x', '') + await this.markRavAsRedeemed(allocationId, senderAddress) logger.info( - `Updated receipt aggregate vouchers table with redeemed_at for allocation ${addressWithoutPrefix}`, + `Updated receipt aggregate vouchers table with redeemed_at for allocation ${allocationId} and sender ${senderAddress}`, ) } catch (err) { logger.warn( @@ -1005,6 +1062,22 @@ export class AllocationReceiptCollector implements ReceiptCollector { ) } + private async markRavAsRedeemed( + allocationId: string, + senderAddress: string, + timestamp?: number, + ) { + // WE use sql directly due to a bug in sequelize update: + // https://github.com/sequelize/sequelize/issues/7664 (bug been open for 7 years no fix yet or ever) + const query = ` + UPDATE scalar_tap_ravs + SET redeemed_at = ${timestamp ? timestamp : 'NOW()'} + WHERE (allocation_id, sender_address) IN '${allocationId}' + AND sender_address = '${senderAddress}' + ` + await this.models.receiptAggregateVouchers.sequelize?.query(query) + } + public async queuePendingReceiptsFromDatabase(): Promise { // Obtain all closed allocations const closedAllocations = await this.models.allocationSummaries.findAll({