From d92d53fa017ca091859a2d2e2763c866cf6970b1 Mon Sep 17 00:00:00 2001 From: Mathieu Lefebvre <49442766+Matlefebvre1234@users.noreply.github.com> Date: Fri, 1 Dec 2023 09:56:54 -0500 Subject: [PATCH 1/3] balance change --- src/fetch/balance.ts | 13 ++++- src/queries.ts | 129 ++++++++++++++++++++++++++++--------------- 2 files changed, 95 insertions(+), 47 deletions(-) diff --git a/src/fetch/balance.ts b/src/fetch/balance.ts index 5260708..2fc4918 100644 --- a/src/fetch/balance.ts +++ b/src/fetch/balance.ts @@ -5,12 +5,23 @@ import { getBalanceChanges } from "../queries.js"; import * as prometheus from "../prometheus.js"; import { toJSON } from "./utils.js"; + +function verifyParams(searchParams: URLSearchParams) { + const chain = searchParams.get("chain"); + const owner = searchParams.get("owner"); + const contract = searchParams.get("contract"); + + if (!chain) throw new Error("chain is required"); + if (!owner && !contract) throw new Error("owner or contract is required"); +} + export default async function (req: Request) { try { const { searchParams } = new URL(req.url); logger.info({ searchParams: Object.fromEntries(Array.from(searchParams)) }); + //Verify required params + verifyParams(searchParams); const query = await getBalanceChanges(searchParams); - const response = await makeQuery(query) return toJSON(response.data); } catch (e: any) { diff --git a/src/queries.ts b/src/queries.ts index 384886f..c707f50 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -32,6 +32,45 @@ export function addAmountFilter(searchParams: URLSearchParams, where: any[]) { } +function balance_changes_owner_contract_query(table: string) { + let query = `SELECT + contract as contract, + owner as owner, + new_balance as balance, + toDateTime(timestamp) as timestamp, + transaction_id as transaction_id, + chain as chain, + block_number`; + + query += ` FROM ${table}` + return query; +} + +function balance_changes_owner_query(table: string) { + let query = `SELECT + owner, + contract, + toDateTime(last_value(timestamp)) AS timestamp, + last_value(new_balance) AS balance`; + + query += ` FROM ${table}` + return query; +} + +function balance_changes_contract_query(table: string) { + let query = `SELECT + owner, + contract, + toDateTime(last_value(timestamp)) as timestamp, + last_value(new_balance) as balance`; + + query += ` FROM ${table}` + return query; +} + + + + export function getTotalSupply(searchParams: URLSearchParams, example?: boolean) { // Params const address = getAddress(searchParams, "address", false)?.toLowerCase(); @@ -44,15 +83,15 @@ export function getTotalSupply(searchParams: URLSearchParams, example?: boolean) const contractTable = 'Contracts'; let query = `SELECT ${table}.address as address, - ${table}.supply as supply, - ${table}.id as id, - block_number, - ${table}.module_hash as module_hash, - ${table}.chain as chain, - ${contractTable}.name as name, - ${contractTable}.symbol as symbol, - ${contractTable}.decimals as decimals, - timestamp + ${table}.supply as supply, + ${table}.id as id, + block_number, + ${table}.module_hash as module_hash, + ${table}.chain as chain, + ${contractTable}.name as name, + ${contractTable}.symbol as symbol, + ${contractTable}.decimals as decimals, + timestamp FROM ${table} `; @@ -74,7 +113,7 @@ export function getTotalSupply(searchParams: URLSearchParams, example?: boolean) // Join WHERE statements with AND - if (where.length) query += ` WHERE (${where.join(' AND ')})`; + if (where.length) query += ` WHERE(${where.join(' AND ')})`; // Sort and Limit const sort_by = searchParams.get("sort_by"); @@ -114,7 +153,7 @@ export function getContracts(searchParams: URLSearchParams, example?: boolean) { addTimestampBlockFilter(searchParams, where); // Join WHERE statements with AND - if (where.length) query += ` WHERE (${where.join(' AND ')})`; + if (where.length) query += ` WHERE(${where.join(' AND ')})`; // Sort and Limit const sort_by = searchParams.get("sort_by"); @@ -133,26 +172,20 @@ export function getBalanceChanges(searchParams: URLSearchParams, example?: boole const contract = getAddress(searchParams, "contract", false)?.toLowerCase(); const owner = getAddress(searchParams, "owner", false)?.toLowerCase(); + + let table; + let contractTable; + let mvOwnerTable = "mv_balance_changes_owner" + let mvContractTable = "mv_balance_changes_contract" + let query = ""; + // SQL Query - const table = 'balance_changes' - const contractTable = 'Contracts'; - let query = `SELECT - ${table}.contract as contract, - ${contractTable}.name as name, - ${contractTable}.symbol as symbol, - ${contractTable}.decimals as decimals, - ${table}.owner as owner, - ${table}.old_balance as old_balance, - ${table}.new_balance as new_balance, - ${table}.transaction_id as transaction_id, - ${table}.id as id, - ${table}.module_hash as module_hash, - ${table}.chain as chain, - block_number, - timestamp - FROM ${table} `; + table = 'balance_changes' + contractTable = 'Contracts'; - query += ` LEFT JOIN Contracts ON ${contractTable}.address = ${table}.contract`; + if (contract && owner) query += balance_changes_owner_contract_query(mvOwnerTable); + if (!contract && owner) query += balance_changes_owner_query(mvContractTable); + if (contract && !owner) query += balance_changes_contract_query(mvContractTable); if (!example) { // WHERE statements @@ -168,13 +201,17 @@ export function getBalanceChanges(searchParams: URLSearchParams, example?: boole addTimestampBlockFilter(searchParams, where); // Join WHERE statements with AND - if (where.length) query += ` WHERE (${where.join(' AND ')})`; + if (where.length) query += ` WHERE(${where.join(' AND ')})`; - // Sort and Limit - const sort_by = searchParams.get("sort_by"); - query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY} ` + + //add ORDER BY and GROUP BY + if (contract && owner) query += ` ORDER BY timestamp DESC` + if (!contract && owner) query += `GROUP BY (contract, owner) ORDER BY timestamp DESC` + if (contract && !owner) query += `GROUP BY (contract, owner) ORDER BY timestamp DESC` } - const limit = parseLimit(searchParams.get("limit"), 100); + + //ADD limit + const limit = parseLimit(searchParams.get("limit")); query += ` LIMIT ${limit} ` const offset = searchParams.get("offset"); if (offset) query += ` OFFSET ${offset} ` @@ -191,15 +228,15 @@ export function getHolders(searchParams: URLSearchParams, example?: boolean) { const table = 'balance_changes' let query = `SELECT owner, - new_balance, - block_number + new_balance, + block_number FROM ${table} `; if (!example) { // WHERE statements const where: any = []; //Get holders balance - let holderWhereQuery = `(owner,block_number) IN (SELECT owner, max(block_number) FROM ${table}`; + let holderWhereQuery = `(owner, block_number) IN(SELECT owner, max(block_number) FROM ${table}`; const whereHolder: any = []; addTimestampBlockFilter(searchParams, whereHolder); if (whereHolder.length) holderWhereQuery += ` WHERE(${whereHolder.join(' AND ')})`; @@ -239,15 +276,15 @@ export function getTransfers(searchParams: URLSearchParams, example?: boolean) { // Query const table = 'Transfers' - let query = `SELECT + let query = `SELECT address as contract, - from, - to, - value as amount, - transaction as transaction_id, - block_number, - timestamp, - chain + from, + to, + value as amount, + transaction as transaction_id, + block_number, + timestamp, + chain FROM ${table} `; if (!example) { @@ -267,7 +304,7 @@ export function getTransfers(searchParams: URLSearchParams, example?: boolean) { addTimestampBlockFilter(searchParams, where); // Join WHERE statements with AND - if (where.length) query += ` WHERE (${where.join(' AND ')})`; + if (where.length) query += ` WHERE(${where.join(' AND ')})`; // Sort and Limit const sort_by = searchParams.get("sort_by"); From ecd8381e656ce69c785051d4d3bac47ddb3ec63f Mon Sep 17 00:00:00 2001 From: Mathieu Lefebvre <49442766+Matlefebvre1234@users.noreply.github.com> Date: Fri, 1 Dec 2023 10:55:56 -0500 Subject: [PATCH 2/3] holder and graph_out refactoring --- src/queries.ts | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/queries.ts b/src/queries.ts index c707f50..b20c539 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -36,9 +36,9 @@ function balance_changes_owner_contract_query(table: string) { let query = `SELECT contract as contract, owner as owner, - new_balance as balance, + newBalance as balance, toDateTime(timestamp) as timestamp, - transaction_id as transaction_id, + transaction as transaction_id, chain as chain, block_number`; @@ -51,7 +51,7 @@ function balance_changes_owner_query(table: string) { owner, contract, toDateTime(last_value(timestamp)) AS timestamp, - last_value(new_balance) AS balance`; + last_value(newBalance) AS balance`; query += ` FROM ${table}` return query; @@ -62,7 +62,7 @@ function balance_changes_contract_query(table: string) { owner, contract, toDateTime(last_value(timestamp)) as timestamp, - last_value(new_balance) as balance`; + last_value(newBalance) as balance`; query += ` FROM ${table}` return query; @@ -180,7 +180,7 @@ export function getBalanceChanges(searchParams: URLSearchParams, example?: boole let query = ""; // SQL Query - table = 'balance_changes' + table = 'BalanceChange' contractTable = 'Contracts'; if (contract && owner) query += balance_changes_owner_contract_query(mvOwnerTable); @@ -225,30 +225,20 @@ export function getHolders(searchParams: URLSearchParams, example?: boolean) { const owner = getAddress(searchParams, "owner", false)?.toLowerCase(); const transaction_id = searchParams.get("transaction_id")?.toLowerCase(); // SQL Query - const table = 'balance_changes' + const table = 'mv_balance_changes_contract' let query = `SELECT owner, - new_balance, - block_number + newBalance AS balance, + block_number, + toDateTime(timestamp) AS timestamp FROM ${table} `; if (!example) { // WHERE statements const where: any = []; - //Get holders balance - let holderWhereQuery = `(owner, block_number) IN(SELECT owner, max(block_number) FROM ${table}`; - const whereHolder: any = []; - addTimestampBlockFilter(searchParams, whereHolder); - if (whereHolder.length) holderWhereQuery += ` WHERE(${whereHolder.join(' AND ')})`; - holderWhereQuery += ` GROUP BY owner)`; - - where.push(holderWhereQuery); - - // equals - if (chain) where.push(`chain == '${chain}'`); if (contract) where.push(`contract == '${contract}'`); - where.push(`CAST(new_balance as int) > 0`); + where.push(`CAST(balance as int) > 0`); // timestamp and block filter addTimestampBlockFilter(searchParams, where); @@ -256,6 +246,8 @@ export function getHolders(searchParams: URLSearchParams, example?: boolean) { // Join WHERE statements with AND if (where.length) query += ` WHERE(${where.join(' AND ')})`; + //add ORDER BY and GROUP BY + query += `ORDER BY timestamp DESC` } const limit = parseLimit(searchParams.get("limit"), 100); From 4575acf03efcf9cc04b42efae85658f6084bb9c8 Mon Sep 17 00:00:00 2001 From: Mathieu Lefebvre <49442766+Matlefebvre1234@users.noreply.github.com> Date: Fri, 1 Dec 2023 12:18:26 -0500 Subject: [PATCH 3/3] transfers optimisation --- src/queries.spec.ts | 145 +++++++++++++++++++++++++++++--------------- src/queries.ts | 49 +++++++++------ 2 files changed, 128 insertions(+), 66 deletions(-) diff --git a/src/queries.spec.ts b/src/queries.spec.ts index 3eb9707..69390db 100644 --- a/src/queries.spec.ts +++ b/src/queries.spec.ts @@ -162,43 +162,68 @@ test("getTotalSupply with options", () => { // Test Balance Change test("getBalanceChange", () => { - const parameters = new URLSearchParams({ chain, owner: address }); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL(`SELECT balance_changes.contract as contract, - Contracts.name as name, - Contracts.symbol as symbol, - Contracts.decimals as decimals, - balance_changes.owner as owner, - balance_changes.old_balance as old_balance, - balance_changes.new_balance as new_balance, - balance_changes.transaction_id as transaction_id, - balance_changes.id as id, - balance_changes.module_hash as module_hash, - balance_changes.chain as chain, - block_number, - timestamp`) + const parameters1 = new URLSearchParams({ chain, owner: address, contract: address }); + const parameters2 = new URLSearchParams({ chain, owner: address }); + const parameters3 = new URLSearchParams({ chain, contract: address }); + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`SELECT + contract as contract, + owner as owner, + newBalance as balance, + toDateTime(timestamp) as timestamp, + transaction as transaction_id, + chain as chain, + block_number`) ); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL(`FROM balance_changes`) + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`SELECT + owner, + contract, + toDateTime(last_value(timestamp)) AS timestamp, + last_value(newBalance) AS balance`) + ); + + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`SELECT + owner, + contract, + toDateTime(last_value(timestamp)) as timestamp, + last_value(newBalance) as balance`) ); + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`FROM mv_balance_changes_owner`) + ); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL( - `LEFT JOIN Contracts ON Contracts.address = balance_changes.contract` - ) + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`FROM mv_balance_changes_contract`) ); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL(`WHERE(chain == '${chain}' AND owner == '${address}')`) + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`FROM mv_balance_changes_contract`) ); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL(`ORDER BY block_number DESC`) + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`WHERE(chain == '${chain}' AND owner == '${address}' AND contract == '${address}')`) ); - expect(formatSQL(getBalanceChanges(parameters))).toContain( - formatSQL(`LIMIT 100`) + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getBalanceChanges(parameters2))).toContain( + formatSQL(`GROUP BY (contract, owner) ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getBalanceChanges(parameters3))).toContain( + formatSQL(`GROUP BY (contract, owner) ORDER BY timestamp DESC`) + ); + + + expect(formatSQL(getBalanceChanges(parameters1))).toContain( + formatSQL(`LIMIT 1`) ); }); @@ -221,32 +246,52 @@ test("getBalanceChanges with options", () => { // Test getTransfers test("getTransfers", () => { - const parameters = new URLSearchParams({ chain, contract: address, from: address, to: address, transaction_id }); - expect(formatSQL(getTransfers(parameters))).toContain( - formatSQL(`SELECT + const parameters1 = new URLSearchParams({ chain, contract: address }); + const parameters2 = new URLSearchParams({ chain, from: address }); + const parameters3 = new URLSearchParams({ chain, to: address }); + const parameters4 = new URLSearchParams({ chain, from: address, to: address }); + const parameters5 = new URLSearchParams({ chain, contract: address, from: address, to: address, transaction_id }); + expect(formatSQL(getTransfers(parameters1))).toContain( + formatSQL(`SELECT address as contract, - from, - to, - value as amount, - transaction as transaction_id, - block_number, - timestamp, - chain`) + from, + to, + value as amount, + transaction as transaction_id, + block_number, + timestamp, + chain`) + ); - expect(formatSQL(getTransfers(parameters))).toContain( - formatSQL(`FROM Transfers`) + expect(formatSQL(getTransfers(parameters1))).toContain( + formatSQL(`FROM mv_transfers_contract`) ); + expect(formatSQL(getTransfers(parameters2))).toContain( + formatSQL(`FROM mv_transfers_from`) + ); - expect(formatSQL(getTransfers(parameters))).toContain( - formatSQL(`WHERE(Transfers.chain == '${chain}' AND Transfers.address == '${address}' AND Transfers.from == '${address}' AND Transfers.to == '${address}' AND Transfers.transaction == '${transaction_id}')`) + expect(formatSQL(getTransfers(parameters3))).toContain( + formatSQL(`FROM mv_transfers_to`) ); - expect(formatSQL(getTransfers(parameters))).toContain( - formatSQL(`ORDER BY block_number DESC`) + expect(formatSQL(getTransfers(parameters4))).toContain( + formatSQL(`FROM mv_transfers_from`) ); - expect(formatSQL(getTransfers(parameters))).toContain( + + + + + expect(formatSQL(getTransfers(parameters5))).toContain( + formatSQL(`WHERE(chain == '${chain}' AND address == '${address}' AND from == '${address}' AND to == '${address}' AND transaction == '${transaction_id}')`) + ); + + expect(formatSQL(getTransfers(parameters5))).toContain( + formatSQL(`ORDER BY timestamp DESC`) + ); + + expect(formatSQL(getTransfers(parameters5))).toContain( formatSQL(`LIMIT 100`) ); }); @@ -255,19 +300,23 @@ test("getTransfers", () => { test("getHolders", () => { const parameters = new URLSearchParams({ chain, contract: address }); expect(formatSQL(getHolders(parameters))).toContain( - formatSQL(`SELECT owner`) + formatSQL(`SELECT + owner, + newBalance AS balance, + block_number, + toDateTime(timestamp) AS timestamp`) ); expect(formatSQL(getHolders(parameters))).toContain( - formatSQL(`FROM balance_changes`) + formatSQL(`FROM mv_balance_changes_contract`) ); expect(formatSQL(getHolders(parameters))).toContain( - formatSQL(`WHERE ((owner,block_number)IN(SELECTowner,max(block_number)FROMbalance_changesGROUPBYowner) AND chain == '${chain}' AND contract == '${address}' AND CAST(new_balance as int) > 0)`) + formatSQL(`WHERE (chain == '${chain}' AND contract == '${address}' AND CAST(balance as int) > 0)`) ); expect(formatSQL(getHolders(parameters))).toContain( - formatSQL(`GROUP BY owner`) + formatSQL(`ORDER BY timestamp DESC`) ); expect(formatSQL(getHolders(parameters))).toContain( diff --git a/src/queries.ts b/src/queries.ts index b20c539..e91f6f4 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -71,6 +71,11 @@ function balance_changes_contract_query(table: string) { + + + + + export function getTotalSupply(searchParams: URLSearchParams, example?: boolean) { // Params const address = getAddress(searchParams, "address", false)?.toLowerCase(); @@ -181,11 +186,11 @@ export function getBalanceChanges(searchParams: URLSearchParams, example?: boole // SQL Query table = 'BalanceChange' - contractTable = 'Contracts'; + if (contract && owner) query += balance_changes_owner_contract_query(mvOwnerTable); - if (!contract && owner) query += balance_changes_owner_query(mvContractTable); - if (contract && !owner) query += balance_changes_contract_query(mvContractTable); + else if (!contract && owner) query += balance_changes_owner_query(mvContractTable); + else if (contract && !owner) query += balance_changes_contract_query(mvContractTable); if (!example) { // WHERE statements @@ -258,15 +263,19 @@ export function getHolders(searchParams: URLSearchParams, example?: boolean) { } export function getTransfers(searchParams: URLSearchParams, example?: boolean) { - // Params + const contract = getAddress(searchParams, "contract", false)?.toLowerCase(); const from = getAddress(searchParams, "from", false)?.toLowerCase(); const to = getAddress(searchParams, "to", false)?.toLowerCase(); const chain = searchParams.get("chain"); const transaction_id = searchParams.get("transaction_id")?.toLowerCase(); const amount = searchParams.get("amount"); - // Query - const table = 'Transfers' + + + // SQL Query + let mvFromTable = "mv_transfers_from" + let mvToTable = "mv_transfers_to" + let mvContractTable = "mv_transfers_contract" let query = `SELECT address as contract, @@ -276,19 +285,24 @@ export function getTransfers(searchParams: URLSearchParams, example?: boolean) { transaction as transaction_id, block_number, timestamp, - chain - FROM ${table} `; + chain` + + if (contract) query += ` FROM ${mvContractTable}` + else if (!contract && from && !to) query += ` FROM ${mvFromTable}` + else if (!contract && !from && to) query += ` FROM ${mvToTable}` + else if (!contract && from && to) query += ` FROM ${mvFromTable}` + if (!example) { // WHERE statements const where = []; // equals - if (chain) where.push(`${table}.chain == '${chain}'`); - if (contract) where.push(`${table}.address == '${contract}'`); - if (from) where.push(`${table}.from == '${from}'`); - if (to) where.push(`${table}.to == '${to}'`); - if (transaction_id) where.push(`${table}.transaction == '${transaction_id}'`); + if (chain) where.push(`chain == '${chain}'`); + if (contract) where.push(`address == '${contract}'`); + if (from) where.push(`from == '${from}'`); + if (to) where.push(`to == '${to}'`); + if (transaction_id) where.push(`transaction == '${transaction_id}'`); //add amount filter addAmountFilter(searchParams, where); @@ -297,12 +311,11 @@ export function getTransfers(searchParams: URLSearchParams, example?: boolean) { // Join WHERE statements with AND if (where.length) query += ` WHERE(${where.join(' AND ')})`; - - // Sort and Limit - const sort_by = searchParams.get("sort_by"); - query += ` ORDER BY block_number ${sort_by ?? DEFAULT_SORT_BY} ` - + //add ORDER BY and GROUP BY + query += ` ORDER BY timestamp DESC` } + + //ADD limit const limit = parseLimit(searchParams.get("limit"), 100); query += ` LIMIT ${limit} ` const offset = searchParams.get("offset");