Feature/subtivity (#51)

* Added aggregation endpoints

* Added /uaw endpoint

* Added supported chains as store (periodically updated)

* Added chain parameter verification

* Simplified queries and tests

* Added /uaw/history endpoint

* fixed endpoint name and unix date to seconds

* enum time range filter and results split by chain

---------

Co-authored-by: Pelotfr <polomeignan@hotmail.fr>

README.md

@@ -10,6 +10,10 @@
 |-------------------------------------------|-----------------------|
 | GET `/chains`                             | Available `chains`
 | GET `/block`                              | Get block by `block_number`, `block_id` or `timestamp`
+| GET `/trace_calls`                        | Get aggregate of trace_calls filtered by `chain`, `timestamp` or `block_number`
+| GET `/transaction_traces`                 | Get aggregate of transaction_traces filtered by `chain`, `timestamp` or `block_number`
+| GET `/uaw`                                | Get unique active wallets filtered by `chain` and `date`
+| GET `/uaw/history`                        | Get daily unique active wallets for previous given number of days filtered by `chain`
 | GET `/health`                             | Health check
 | GET `/metrics`                            | Prometheus metrics
 | GET `/openapi`                            | [OpenAPI v3 JSON](https://spec.openapis.org/oas/v3.0.0)

src/clickhouse/stores.ts

@@ -0,0 +1,29 @@
+import client from "./createClient.js";
+
+class ClickhouseStore {
+  private ChainsPromise: Promise<string[]> | undefined = undefined;
+
+  constructor() {
+    // Fetch data initially
+    this.fetchData();
+
+    // Set up a timer to fetch data periodically (e.g., every 1 hour)
+    setInterval(() => {
+      this.fetchData();
+    }, 10000); // 3600000 milliseconds = 1 hour
+  }
+
+  private fetchData() {
+    this.ChainsPromise = client
+      .query({ query: "SELECT DISTINCT chain FROM blocks", format: "JSONEachRow" })
+      .then((response) => response.json<Array<{ chain: string }>>())
+      .then((chains) => chains.map(({ chain }) => chain))
+      .catch(() => []);
+  }
+
+  public get chains() {
+    return this.ChainsPromise;
+  }
+}
+
+export const store = new ClickhouseStore();

src/config.ts

@@ -15,6 +15,7 @@ export const DEFAULT_MAX_LIMIT = 500;
 export const DEFAULT_VERBOSE = false;
 export const APP_NAME = pkg.name;
 export const DEFAULT_SORT_BY = "DESC";
+export const DEFAULT_AGGREGATE_FUNCTION = "count";
 
 // parse command line options
 const opts = program

src/fetch/GET.ts

@@ -3,6 +3,9 @@ import openapi from "./openapi.js";
 import health from "./health.js";
 import chains from "./chains.js";
 import block from "./block.js";
+import aggregate from "./aggregate.js";
+import uaw from "./uaw.js";
+import history from "./history.js";
 import * as prometheus from "../prometheus.js";
 import { logger } from "../logger.js";
 import swaggerHtml from "../../swagger/index.html"
@@ -17,8 +20,12 @@ export default async function (req: Request) {
     if ( pathname === "/health" ) return health(req);
     if ( pathname === "/metrics" ) return toText(await registry.metrics());
     if ( pathname === "/openapi" ) return toJSON(openapi);
-    if ( pathname === "/chains" ) return chains(req);
+    if ( pathname === "/chains" ) return chains();
     if ( pathname === "/block" ) return block(req);
+    if ( pathname === "/trace_calls" ) return aggregate(req, pathname);
+    if ( pathname === "/transaction_traces" ) return aggregate(req, pathname);
+    if ( pathname === "/uaw" ) return uaw(req);
+    if ( pathname === "/uaw/history" ) return history(req);
     logger.warn(`Not found: ${pathname}`);
     prometheus.request_error.inc({pathname, status: 404});
     return NotFound;

src/fetch/aggregate.ts

@@ -0,0 +1,24 @@
+import { makeQuery } from "../clickhouse/makeQuery.js";
+import { logger } from "../logger.js";
+import { getAggregate } from "../queries.js";
+import * as prometheus from "../prometheus.js";
+import { BadRequest, toJSON } from "./cors.js";
+import { verifyParameters } from "../utils.js";
+
+export default async function (req: Request, pathname: string) {
+  const parametersResult = await verifyParameters(req);
+  if(parametersResult instanceof Response) {
+    return parametersResult;
+  }
+  try {
+    const { searchParams } = new URL(req.url);
+    logger.info({searchParams: Object.fromEntries(Array.from(searchParams))});
+    const query = getAggregate(searchParams, pathname.replace("/", ""));
+    const response = await makeQuery<number>(query)
+    return toJSON(response.data);
+  } catch (e: any) {
+    logger.error(e);
+    prometheus.request_error.inc({pathname: pathname, status: 400});
+    return BadRequest
+  }
+}

src/fetch/block.ts

@@ -3,8 +3,13 @@ import { logger } from "../logger.js";
 import { Block, getBlock } from "../queries.js";
 import * as prometheus from "../prometheus.js";
 import { BadRequest, toJSON } from "./cors.js";
+import { verifyParameters } from "../utils.js";
 
 export default async function (req: Request) {
+  const parametersResult = await verifyParameters(req);
+  if(parametersResult instanceof Response) {
+    return parametersResult;
+  }
   try {
     const { searchParams } = new URL(req.url);
     logger.info({searchParams: Object.fromEntries(Array.from(searchParams))});

src/fetch/chains.ts

@@ -1,17 +1,11 @@
-import { makeQuery } from "../clickhouse/makeQuery.js";
 import { logger } from "../logger.js";
+import { store } from "../clickhouse/stores.js";
 import * as prometheus from "../prometheus.js";
-import { getChain } from "../queries.js";
 import { BadRequest, toJSON } from "./cors.js";
 
-export async function supportedChainsQuery() {
-  const response = await makeQuery<{chain: string}>(getChain());
-  return response.data.map((r) => r.chain);
-}
-
-export default async function (req: Request) {
+export default async function () {
   try {
-    const chains = await supportedChainsQuery();
+    const chains = await store.chains;
     return toJSON(chains);
   } catch (e: any) {
     logger.error(e);

src/fetch/history.ts

@@ -0,0 +1,25 @@
+import { makeQuery } from "../clickhouse/makeQuery.js";
+import { logger } from "../logger.js";
+import { UAWHistory, getUAWHistory } from "../queries.js";
+import * as prometheus from "../prometheus.js";
+import { BadRequest, toJSON } from "./cors.js";
+import { parseUAWResponse, verifyParameters } from "../utils.js";
+
+export default async function (req: Request) {
+  const parametersResult = await verifyParameters(req);
+  if(parametersResult instanceof Response) {
+    return parametersResult;
+  }
+  try {
+    const { searchParams } = new URL(req.url);
+    logger.info({searchParams: Object.fromEntries(Array.from(searchParams))});
+    const query = getUAWHistory(searchParams);
+    const response = await makeQuery<UAWHistory>(query)
+    const formatted = parseUAWResponse(response.data);
+    return toJSON(formatted);
+  } catch (e: any) {
+    logger.error(e);
+    prometheus.request_error.inc({pathname: "/uaw/history", status: 400});
+    return BadRequest
+  }
+}