From bfbd63c5a53a786e26d7164146eb8dc1fceda3ad Mon Sep 17 00:00:00 2001
From: Theo Butler <theodusbutler@gmail.com>
Date: Thu, 29 Feb 2024 09:44:33 -0500
Subject: [PATCH 1/2] fix: paginate allocation queries

---
 packages/indexer-service/src/allocations.ts | 96 ++++++++++++++-------
 1 file changed, 67 insertions(+), 29 deletions(-)

diff --git a/packages/indexer-service/src/allocations.ts b/packages/indexer-service/src/allocations.ts
index c002b0122..9345f7b18 100644
--- a/packages/indexer-service/src/allocations.ts
+++ b/packages/indexer-service/src/allocations.ts
@@ -61,13 +61,16 @@ export const monitorEligibleAllocations = ({
 
       const currentEpoch = currentEpochResult.data.graphNetwork.currentEpoch
 
-      const result = await networkSubgraph.query(
-        gql`
-          query allocations($indexer: String!, $closedAtEpochThreshold: Int!) {
-            indexer(id: $indexer) {
-              activeAllocations: totalAllocations(
-                where: { status: Active }
-                orderDirection: desc
+      let lastId = ''
+      const activeAllocations = []
+      for (;;) {
+        const result = await networkSubgraph.query(
+          gql`
+            query allocations($indexer: String!, $lastId: String!) {
+              allocations(
+                where: { indexer: $indexer, id_gt: $lastId, status: Active }
+                orderBy: id
+                orderDirection: asc
                 first: 1000
               ) {
                 id
@@ -85,9 +88,43 @@ export const monitorEligibleAllocations = ({
                   queryFeesAmount
                 }
               }
-              recentlyClosedAllocations: totalAllocations(
-                where: { status: Closed, closedAtEpoch_gte: $closedAtEpochThreshold }
-                orderDirection: desc
+            }
+          `,
+          {
+            indexer: indexer.toLowerCase(),
+            lastId,
+          },
+        )
+
+        if (result.error) {
+          throw result.error
+        }
+        if (result.data.allocations.length == 0) {
+          break
+        }
+        activeAllocations.push(...result.data.allocations)
+        lastId = result.data.allocations.slice(-1)[0].id
+      }
+
+      lastId = ''
+      const recentlyClosedAllocations = []
+      for (;;) {
+        const result = await networkSubgraph.query(
+          gql`
+            query allocations(
+              $indexer: String!
+              $lastId: String!
+              $closedAtEpochThreshold: Int!
+            ) {
+              allocations(
+                where: {
+                  indexer: $indexer
+                  id_gt: $lastId
+                  status: Closed
+                  closedAtEpoch_gte: $closedAtEpochThreshold
+                }
+                orderBy: id
+                orderDirection: asc
                 first: 1000
               ) {
                 id
@@ -106,31 +143,32 @@ export const monitorEligibleAllocations = ({
                 }
               }
             }
-          }
-        `,
-        {
-          indexer: indexer.toLowerCase(),
-          closedAtEpochThreshold: currentEpoch - 1, // allocation can be closed within the last epoch or later
-        },
-      )
-
-      if (result.error) {
-        throw result.error
-      }
+          `,
+          {
+            indexer: indexer.toLowerCase(),
+            lastId,
+            closedAtEpochThreshold: currentEpoch - 1, // allocation can be closed within the last epoch or later
+          },
+        )
 
-      if (!result.data) {
-        throw new Error(`No data / indexer not found on chain`)
+        if (result.error) {
+          throw result.error
+        }
+        if (result.data.allocations.length == 0) {
+          break
+        }
+        recentlyClosedAllocations.push(...result.data.allocations)
+        lastId = result.data.allocations.slice(-1)[0].id
       }
 
-      if (!result.data.indexer) {
-        throw new Error(`Indexer not found on chain`)
+      const allocations = [...activeAllocations, ...recentlyClosedAllocations]
+
+      if (allocations.length == 0) {
+        throw new Error(`No data / indexer not found on chain`)
       }
 
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      return [
-        ...result.data.indexer.activeAllocations,
-        ...result.data.indexer.recentlyClosedAllocations,
-      ].map(x => parseGraphQLAllocation(x, protocolNetwork))
+      return allocations.map(x => parseGraphQLAllocation(x, protocolNetwork))
     } catch (err) {
       logger.warn(`Failed to query indexer allocations, keeping existing`, {
         allocations: currentAllocations.map(allocation => allocation.id),

From 5d43368a0ebbfd7f44f0613457544ecad2b539ed Mon Sep 17 00:00:00 2001
From: Ariel Barmat <140773+abarmat@users.noreply.github.com>
Date: Thu, 29 Feb 2024 12:40:57 -0300
Subject: [PATCH 2/2] indexer-service: increase the signer cache size and print
 the number loaded

---
 packages/indexer-service/src/allocations.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/indexer-service/src/allocations.ts b/packages/indexer-service/src/allocations.ts
index 9345f7b18..406199aad 100644
--- a/packages/indexer-service/src/allocations.ts
+++ b/packages/indexer-service/src/allocations.ts
@@ -218,7 +218,7 @@ export const ensureAttestationSigners = ({
   const logger = parentLogger.child({ component: 'AttestationSignerCache' })
 
   const cache: AttestationSignerCache = new LRUCache(null, {
-    maxlen: 1000,
+    maxlen: 5000,
   })
 
   const signers = allocations.map(async allocations => {
@@ -263,8 +263,9 @@ export const ensureAttestationSigners = ({
   })
 
   signers.pipe(signers => {
-    logger.info(`Cached attestation signers`, {
-      allocations: [...signers.keys()],
+    const attestationSigners = [...signers.keys()]
+    logger.info(`Cached ${attestationSigners.length} attestation signers`, {
+      allocations: attestationSigners,
     })
   })