From 256fe63bfd5a226917b57cd4321b4945658b7d06 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Sun, 21 Jan 2024 14:30:43 +0000 Subject: [PATCH] test(benchmarks): add benchmarks for e2e actions --- .github/workflows/benchmarks.yml | 16 + circuits/ts/proofs.ts | 4 + cli/package.json | 1 + cli/tests/e2e/e2e.test.ts | 2 +- integrationTests/package.json | 2 + .../ts/__benchmarks__/benchmarks.ts | 367 ++++++++++++++++++ .../ts/__tests__/utils/constants.ts | 13 + pnpm-lock.yaml | 3 + 8 files changed, 407 insertions(+), 1 deletion(-) create mode 100644 integrationTests/ts/__benchmarks__/benchmarks.ts diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 46df463d83..64481e276d 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -33,5 +33,21 @@ jobs: run: | pnpm build + - name: Download rapidsnark (1c137) + run: | + mkdir -p ~/rapidsnark/build + wget -qO ~/rapidsnark/build/prover https://maci-devops-zkeys.s3.ap-northeast-2.amazonaws.com/rapidsnark-linux-amd64-1c137 + chmod +x ~/rapidsnark/build/prover + + - name: Download circom Binary v2.1.6 + run: | + wget -qO ${{ github.workspace }}/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64 + chmod +x ${{ github.workspace }}/circom + sudo mv ${{ github.workspace }}/circom /bin/circom + + - name: Download zkeys + run: | + pnpm download:test-zkeys + - name: Benchmarks run: pnpm benchmarks diff --git a/circuits/ts/proofs.ts b/circuits/ts/proofs.ts index 78e6a64eb5..2a1a6887e1 100644 --- a/circuits/ts/proofs.ts +++ b/circuits/ts/proofs.ts @@ -51,6 +51,10 @@ export const genProof = async ({ } const { proof, publicSignals } = await groth16.fullProve(inputs, wasmPath, zkeyPath); + + // ensure we clean up all threads spawned by snarkjs + await cleanThreads(); + return { proof, publicSignals }; } diff --git a/cli/package.json b/cli/package.json index 933eedda0e..08aa91b755 100644 --- a/cli/package.json +++ b/cli/package.json @@ -13,6 +13,7 @@ ], "scripts": { "watch": "tsc --watch", + "benchmarks": "ts-node tests/benchmarks/benchmarks.ts", "build": "tsc -p tsconfig.build.json", "postbuild": "cp package.json ./build && mkdir -p ./zkeys", "types": "tsc -p tsconfig.json --noEmit", diff --git a/cli/tests/e2e/e2e.test.ts b/cli/tests/e2e/e2e.test.ts index de9c6dfd59..d659e3f999 100644 --- a/cli/tests/e2e/e2e.test.ts +++ b/cli/tests/e2e/e2e.test.ts @@ -315,7 +315,7 @@ describe("e2e tests", function test() { }); }); - describe("5 signups, 1 message", () => { + describe("9 signups, 1 message", () => { after(() => { cleanVanilla(); }); diff --git a/integrationTests/package.json b/integrationTests/package.json index f7e7ad3b21..c5e3d24290 100644 --- a/integrationTests/package.json +++ b/integrationTests/package.json @@ -11,6 +11,7 @@ "scripts": { "watch": "tsc --watch", "build": "tsc", + "benchmarks": "ts-node ts/__benchmarks__/benchmarks.ts", "types": "tsc -p tsconfig.json --noEmit", "test": "ts-mocha --exit ./ts/__tests__/**.test.ts", "test:integration": "NODE_OPTIONS=--max-old-space-size=4096 ts-mocha --exit ./ts/__tests__/integration.test.ts", @@ -22,6 +23,7 @@ "@types/chai-as-promised": "^7.1.8", "@types/mocha": "^10.0.6", "@types/node": "^20.11.2", + "benny": "^3.7.1", "chai": "^4.3.10", "chai-as-promised": "^7.1.1", "hardhat": "^2.19.2", diff --git a/integrationTests/ts/__benchmarks__/benchmarks.ts b/integrationTests/ts/__benchmarks__/benchmarks.ts new file mode 100644 index 0000000000..4ef5dcde12 --- /dev/null +++ b/integrationTests/ts/__benchmarks__/benchmarks.ts @@ -0,0 +1,367 @@ +import benny from "benny"; +import { genProof } from "maci-circuits"; +import { CircuitInputs, MaciState } from "maci-core"; +import { Keypair, PCommand } from "maci-domainobjs"; + +import { homedir } from "os"; +import path from "path"; + +import { STATE_TREE_DEPTH, duration, maxValues, treeDepths, voiceCreditBalance } from "../__tests__/utils/constants"; + +const NAME = "proof generation"; + +// eslint-disable-next-line @typescript-eslint/require-await +export default async function rune2e(): Promise { + const coordinatorKeypair = new Keypair(); + + const messageBatchSize = 5; + + const users = new Array(5).fill(new Keypair()); + + benny.suite( + NAME, + + benny.add(`wasm genProof - 5 signups, 5 messages`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + const poll = maciState.polls.get(pollId); + + // signup 5 users + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + maciState.signUp(keypair.pubKey, voiceCreditBalance, BigInt(Date.now())); + // First command (valid) + const command = new PCommand( + BigInt(i) + 1n, + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + } + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + wasmPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_js/ProcessMessages_10-2-1-2_test.wasm", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + }), + + benny.add(`rapidsnark genProof - 5 signups, 5 messages`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + const poll = maciState.polls.get(pollId); + + // signup 5 users and submit 5 messages + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + maciState.signUp(keypair.pubKey, voiceCreditBalance, BigInt(Date.now())); + + // First command (valid) + const command = new PCommand( + BigInt(i) + 1n, + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + } + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + rapidsnarkExePath: `${homedir()}/rapidsnark/build/prover`, + witnessExePath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_cpp/ProcessMessages_10-2-1-2_test", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + }), + + benny.add(`wasm genProof - 5 signups, 1 message`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + + const poll = maciState.polls.get(pollId); + + // signup 5 users + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + maciState.signUp(keypair.pubKey, voiceCreditBalance, BigInt(Date.now())); + } + + const keypair = users[0]; + // First command (valid) + const command = new PCommand( + 1n, // BigInt(1), + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + wasmPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_js/ProcessMessages_10-2-1-2_test.wasm", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + }), + + benny.add(`rapidnsark genProof - 5 signups, 1 message`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + + const poll = maciState.polls.get(pollId); + + // signup 5 users + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + maciState.signUp(keypair.pubKey, voiceCreditBalance, BigInt(Date.now())); + } + + const keypair = users[0]; + // First command (valid) + const command = new PCommand( + 1n, // BigInt(1), + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + rapidsnarkExePath: `${homedir()}/rapidsnark/build/prover`, + witnessExePath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_cpp/ProcessMessages_10-2-1-2_test", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + }), + + benny.add(`wasm genProof - 1 signups, 5 messages`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + + const poll = maciState.polls.get(pollId); + + maciState.signUp(users[0].pubKey, voiceCreditBalance, BigInt(Date.now())); + + // submit 5 messages + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + // First command (valid) + const command = new PCommand( + BigInt(i) + 1n, + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + wasmPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_js/ProcessMessages_10-2-1-2_test.wasm", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + } + }), + + benny.add(`rapidnsark genProof - 1 signups, 5 messages`, async () => { + const maciState = new MaciState(STATE_TREE_DEPTH); + const pollId = maciState.deployPoll( + BigInt(duration), + maxValues, + treeDepths, + messageBatchSize, + coordinatorKeypair, + ); + + const poll = maciState.polls.get(pollId); + + maciState.signUp(users[0].pubKey, voiceCreditBalance, BigInt(Date.now())); + + // submit 5 messages + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < users.length; i += 1) { + const keypair = users[i]; + // First command (valid) + const command = new PCommand( + BigInt(i) + 1n, + keypair.pubKey, + 5n, // voteOptionIndex, + 7n, // vote weight + 2n, // nonce + pollId, + ); + + const signature = command.sign(keypair.privKey); + + const ecdhKeypair = new Keypair(); + const sharedKey = Keypair.genEcdhSharedKey(ecdhKeypair.privKey, coordinatorKeypair.pubKey); + const message = command.encrypt(signature, sharedKey); + poll?.publishMessage(message, ecdhKeypair.pubKey); + + while (poll?.hasUnprocessedMessages()) { + const pmInputs = poll.processMessages(pollId) as unknown as CircuitInputs; + + // eslint-disable-next-line no-await-in-loop + await genProof({ + inputs: pmInputs, + useWasm: true, + rapidsnarkExePath: `${homedir()}/rapidsnark/build/prover`, + witnessExePath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test_cpp/ProcessMessages_10-2-1-2_test", + ), + zkeyPath: path.resolve( + __dirname, + "../../../cli/zkeys/ProcessMessages_10-2-1-2_test/ProcessMessages_10-2-1-2_test.0.zkey", + ), + }); + } + } + }), + + benny.cycle(), + benny.complete(), + + benny.save({ folder: "ts/__benchmarks__/results", file: NAME, version: "1.0.0", details: true }), + benny.save({ folder: "ts/__benchmarks__/results", file: NAME, format: "chart.html", details: true }), + benny.save({ folder: "ts/__benchmarks__/results", file: NAME, format: "table.html", details: true }), + ); +} + +(async () => { + await rune2e(); +})(); diff --git a/integrationTests/ts/__tests__/utils/constants.ts b/integrationTests/ts/__tests__/utils/constants.ts index 7ce585f9fb..f3ab674985 100644 --- a/integrationTests/ts/__tests__/utils/constants.ts +++ b/integrationTests/ts/__tests__/utils/constants.ts @@ -42,3 +42,16 @@ export const INT_STATE_TREE_DEPTH = 1; export const MSG_TREE_DEPTH = 2; export const VOTE_OPTION_TREE_DEPTH = 2; export const MSG_BATCH_DEPTH = 1; + +export const voiceCreditBalance = 100n; +export const maxValues = { + maxUsers: 25, + maxMessages: 25, + maxVoteOptions: 25, +}; +export const treeDepths = { + intStateTreeDepth: 2, + messageTreeDepth: 2, + messageTreeSubDepth: 1, + voteOptionTreeDepth: 2, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 28b8c2cee3..d40a970a1a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -438,6 +438,9 @@ importers: '@types/node': specifier: ^20.11.2 version: 20.11.2 + benny: + specifier: ^3.7.1 + version: 3.7.1 chai: specifier: ^4.3.10 version: 4.4.0