From 2fdf055ca7d9d6e5fbfc459aee743b8427a62391 Mon Sep 17 00:00:00 2001 From: ctrlc03 <93448202+ctrlc03@users.noreply.github.com> Date: Sun, 14 Jan 2024 23:11:22 +0000 Subject: [PATCH] feat(circuits): remove zkey-manager in favour of circomkit remove zkey-manager and custom scripts in favour of circomkit, to build circuits and generate testing zkeys. Update docs to reflect the change. --- .github/workflows/nightly.yml | 17 +- .github/workflows/reusable-e2e.yml | 23 +-- circuits/.gitignore | 4 + circuits/circom/circuits.json | 12 -- circuits/package.json | 6 +- circuits/scripts/buildCustomSnarks.sh | 76 --------- circuits/scripts/build_arm.sh | 16 -- circuits/scripts/build_intel.sh | 24 --- circuits/ts/compile.ts | 101 ++++++++++++ circuits/ts/genZkeys.ts | 69 ++++++++ circuits/ts/types.ts | 8 + cli/package.json | 5 +- cli/zkeys.config.yml | 150 ------------------ contracts/package.json | 1 + package.json | 3 + pnpm-lock.yaml | 115 ++------------ .../versioned_docs/version-v1.x/circuits.md | 25 ++- .../version-v1.x/installation.md | 55 ++++--- .../versioned_docs/version-v1.x/testing.md | 23 +-- .../version-v1.x/troubleshooting.md | 5 +- 20 files changed, 278 insertions(+), 460 deletions(-) create mode 100644 circuits/.gitignore delete mode 100755 circuits/scripts/buildCustomSnarks.sh delete mode 100644 circuits/scripts/build_arm.sh delete mode 100644 circuits/scripts/build_intel.sh create mode 100644 circuits/ts/compile.ts create mode 100644 circuits/ts/genZkeys.ts delete mode 100644 cli/zkeys.config.yml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0fa7bc028a..827e53568a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -58,9 +58,9 @@ jobs: - name: Download circom Binary v2.1.6 run: | - mkdir -p /home/runner/work/maci/.local/bin - wget -qO /home/runner/work/maci/.local/bin/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64 - chmod +x /home/runner/work/maci/.local/bin/circom + wget -qO ${{ github.workspace }}/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64 + chmod +x ${{ github.workspace }}/circom + sudo mv ${{ github.workspace }}/circom /bin/circom - name: Create zkeys folder run: | @@ -69,15 +69,8 @@ jobs: - name: Compile Circuits And Generate zkeys run: | - cd circuits - export PATH=$PATH:/home/runner/work/maci/.local/bin - pnpm build-test-circuits-c - mv ./build/test/* ../cli/zkeys/ - cd ../cli - wget -qO zkeys/powersOfTau28_hez_final_20.ptau https://maci-devops-zkeys.s3.ap-northeast-2.amazonaws.com/powersOfTau28_hez_final_20.ptau - pnpm snarkjs groth16 setup ./zkeys/ProcessMessages_10-2-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/ProcessMessages_10-2-1-2_test.0.zkey - pnpm snarkjs groth16 setup ./zkeys/TallyVotes_10-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/TallyVotes_10-1-2_test.0.zkey - pnpm snarkjs groth16 setup ./zkeys/SubsidyPerBatch_10-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/SubsidyPerBatch_10-1-2_test.0.zkey + pnpm build:circuits-c + pnpm setup:zkeys - name: ${{ matrix.command }} run: pnpm run ${{ matrix.command }} diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 07e6372d6c..5d1be76708 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -75,28 +75,15 @@ jobs: - name: Download circom Binary v2.1.6 run: | - mkdir -p /home/runner/work/maci/.local/bin - wget -qO /home/runner/work/maci/.local/bin/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64 - chmod +x /home/runner/work/maci/.local/bin/circom - - - name: Create zkeys folder - if: ${{ env.CHANGED == 'true' }} - run: | - cd cli - mkdir -p zkeys + wget -qO ${{ github.workspace }}/circom https://github.com/iden3/circom/releases/download/v2.1.6/circom-linux-amd64 + chmod +x ${{ github.workspace }}/circom + sudo mv ${{ github.workspace }}/circom /bin/circom - name: Compile Circuits And Generate zkeys if: ${{ env.CHANGED == 'true' }} run: | - cd circuits - export PATH=$PATH:/home/runner/work/maci/.local/bin - pnpm build-test-circuits-c - mv ./build/test/* ../cli/zkeys/ - cd ../cli - wget -qO zkeys/powersOfTau28_hez_final_20.ptau https://maci-devops-zkeys.s3.ap-northeast-2.amazonaws.com/powersOfTau28_hez_final_20.ptau - pnpm snarkjs groth16 setup ./zkeys/ProcessMessages_10-2-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/ProcessMessages_10-2-1-2_test.0.zkey - pnpm snarkjs groth16 setup ./zkeys/TallyVotes_10-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/TallyVotes_10-1-2_test.0.zkey - pnpm snarkjs groth16 setup ./zkeys/SubsidyPerBatch_10-1-2_test.r1cs ./zkeys/powersOfTau28_hez_final_20.ptau ./zkeys/SubsidyPerBatch_10-1-2_test.0.zkey + pnpm build:circuits-c + pnpm setup:zkeys - name: Download zkeys if: ${{ env.CHANGED == 'false' }} diff --git a/circuits/.gitignore b/circuits/.gitignore new file mode 100644 index 0000000000..5375b2d730 --- /dev/null +++ b/circuits/.gitignore @@ -0,0 +1,4 @@ +ptau +input.json +circom/main +circom/test \ No newline at end of file diff --git a/circuits/circom/circuits.json b/circuits/circom/circuits.json index 412f078e02..d3dd5ce2c8 100644 --- a/circuits/circom/circuits.json +++ b/circuits/circom/circuits.json @@ -1,22 +1,10 @@ { - "processMessages": { - "file": "processMessages", - "template": "ProcessMessages", - "params": [6, 8, 2, 3], - "pubs": ["inputHash"] - }, "ProcessMessages_10-2-1-2_test": { "file": "processMessages", "template": "ProcessMessages", "params": [10, 2, 1, 2], "pubs": ["inputHash"] }, - "tallyVotes": { - "file": "tallyVotes", - "template": "TallyVotes", - "params": [6, 2, 3], - "pubs": ["inputHash"] - }, "TallyVotes_10-1-2_test": { "file": "tallyVotes", "template": "TallyVotes", diff --git a/circuits/package.json b/circuits/package.json index 6c84e04acf..b0723fa794 100644 --- a/circuits/package.json +++ b/circuits/package.json @@ -4,8 +4,9 @@ "description": "zk-SNARK circuits for MACI", "main": "build/ts/index.js", "scripts": { - "build-test-circuits-c": "bash ./scripts/build_intel.sh", - "build-test-circuits-wasm": "bash ./scripts/build_arm.sh", + "build-test-circuits-c": "ts-node ./ts/compile.ts --cWitness", + "build-test-circuits-wasm": "ts-node ./ts/compile.ts", + "gen-zkeys": "ts-node ./ts/genZKeys.ts", "watch": "tsc --watch", "build": "tsc -p tsconfig.build.json", "circom:build": "NODE_OPTIONS=--max-old-space-size=4096 circomkit compile", @@ -46,6 +47,7 @@ "chai-as-promised": "^7.1.1", "mocha": "^10.2.0", "ts-mocha": "^10.0.0", + "ts-node": "^10.9.1", "typescript": "^5.3.3" } } diff --git a/circuits/scripts/buildCustomSnarks.sh b/circuits/scripts/buildCustomSnarks.sh deleted file mode 100755 index 45351cde9c..0000000000 --- a/circuits/scripts/buildCustomSnarks.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -e - -rargs=4 -function checkargs { - if [ $# -lt $rargs ] - then - help - echo "Error: missing arguments" >&2 - exit 1 - fi - - # Intermediate state tree depth such that 5^istd > max number of messages - istd=$(echo "l($n)/l(5)" | bc -l ) - - if [ $(echo $istd | cut -d . -f2) -ne "00000000000000000000" ] - then - echo "Error: Max number of messages must have a prime factor of 5" 1>&2 - exit 1 - fi - - istd=$(echo $istd | cut -d . -f1) -} - -help() { - echo "Builds the processMessage and tallyVotes with custom parameters." - echo - echo "Syntax: buildCustomSnarks [-h|s|m|v|i|b|n]" - echo - echo "options: " - echo "s State tree depth" - echo "m Message tree depth" - echo "v Vote options tree depth" - echo "b Message Batch depth (msgSubTreeDepth)" - echo "n Max number of messages" -} - -while getopts ":h:s:m:v:i:b:n:" option; do - case $option in - h) - help - exit;; - s) - std="$OPTARG";; - m) - mtd="$OPTARG";; - v) - votd="$OPTARG";; - b) - b="$OPTARG";; - n) - n="$OPTARG";; - \?) - echo "Error: Invalid option" 1>&2 - exit;; - esac -done - -checkargs $std $mtd $votd $b $n - -cd "$(dirname "$0")" -cd .. - -pcircuitdir="./circom/prod/processMessages_custom.circom" -tcircuitdir="./circom/prod/tallyVotes_custom.circom" -touch $pcircuitdir -touch $tcircuitdir - -# Write custom circuit files -echo -e "pragma circom 2.0.0;" > $pcircuitdir -echo -e 'include "../processMessages.circom";\n' >> $pcircuitdir -echo "component main {public [inputHash]} = ProcessMessages($std, $mtd, $b, $votd);" >> $pcircuitdir - -echo -e "pragma circom 2.0.0;" > $tcircuitdir -echo -e 'include "../tallyVotes.circom";\n' > $tcircuitdir -echo "component main {public [inputHash]} = TallyVotes($std, $istd, $votd);" >> $tcircuitdir diff --git a/circuits/scripts/build_arm.sh b/circuits/scripts/build_arm.sh deleted file mode 100644 index 551ca8928b..0000000000 --- a/circuits/scripts/build_arm.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Assuming this script file is located under circuits/circom/scripts and -CWD=$(pwd) -OUTPUT_PATH="../build/test/" -CIRCUITS_PATH="./circom/test" -mkdir -p "$OUTPUT_PATH" - -for circuit in "$CIRCUITS_PATH"/*.circom -do - # Compile circuit - echo "#### Compile Circuit: "$circuit"" - circom --O2 --wasm --r1cs --sym --output "$OUTPUT_PATH" -l ./node_modules/circomlib/circuits -l ./node_modules/@zk-kit/circuits/circom "$circuit" - - cd "$CWD" -done diff --git a/circuits/scripts/build_intel.sh b/circuits/scripts/build_intel.sh deleted file mode 100644 index 7a60d60442..0000000000 --- a/circuits/scripts/build_intel.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Assuming this script file is located under circuits/circom/scripts and -CWD=$(pwd) -OUTPUT_PATH="./build/test/" -CIRCUITS_PATH="./circom/test" -mkdir -p "$OUTPUT_PATH" - -for circuit in "$CIRCUITS_PATH"/*.circom -do - # Compile circuit with circom directly as we need the c witness file - echo "#### Compile Circuit: "$circuit"" - circom --O2 --wasm --r1cs --c --sym --output "$OUTPUT_PATH" -l ./node_modules/circomlib/circuits -l ./node_modules/@zk-kit/circuits/circom "$circuit" - # Generate executable that can compute the witness of this circuit - filename=$(basename "$circuit" .circom) - cd ""$OUTPUT_PATH""$filename"_cpp" - make - - # Move the witness files out of their folders - mv "$filename" "../" - mv "$filename.dat" "../" - - cd "$CWD" -done diff --git a/circuits/ts/compile.ts b/circuits/ts/compile.ts new file mode 100644 index 0000000000..7600cfc271 --- /dev/null +++ b/circuits/ts/compile.ts @@ -0,0 +1,101 @@ +import { type CircomkitConfig, type CircuitConfig, Circomkit } from "circomkit"; + +import { execSync } from "child_process"; +import fs from "fs"; +import path from "path"; + +import type { CircuitConfigWithName } from "./types"; + +/** + * Compile MACI's circuits using circomkit + * and setup the dir structure + * @param cWitness - whether to compile to the c witness generator + * or not + * @param outputPath - the path to the output folder + * @returns the build directory + */ +export const compileCircuits = async (cWitness?: boolean, outputPath?: string): Promise => { + // read circomkit config files + const configFilePath = path.resolve(__dirname, "..", "circomkit.json"); + const circomKitConfig = JSON.parse(fs.readFileSync(configFilePath, "utf-8")) as CircomkitConfig; + const circuitsConfigPath = path.resolve(__dirname, "..", "circom", "circuits.json"); + const circuitsConfigContent = JSON.parse(fs.readFileSync(circuitsConfigPath, "utf-8")) as unknown as Record< + string, + CircuitConfig + >; + const circuitsConfigs: CircuitConfigWithName[] = Object.entries(circuitsConfigContent).map(([name, config]) => ({ + name, + ...config, + })); + + const outputPathUpdated = outputPath ? path.resolve(outputPath) : undefined; + // set the config based on whether to compile the c witness or no + if (cWitness) { + circomKitConfig.cWitness = true; + } else { + circomKitConfig.cWitness = false; + } + + // update the build directory if we have an output path + if (outputPathUpdated) { + circomKitConfig.dirBuild = outputPathUpdated; + circomKitConfig.dirPtau = outputPathUpdated; + } + + const circomkitInstance = new Circomkit({ + ...circomKitConfig, + verbose: false, + }); + + // loop through each circuit config and compile them + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < circuitsConfigs.length; i += 1) { + const circuit = circuitsConfigs[i]; + // eslint-disable-next-line no-await-in-loop + const outPath = await circomkitInstance.compile(circuit.name, circuit); + + // if the circuit is compiled with a c witness, then let's run make in the directory + if (cWitness) { + try { + // build + execSync(`cd ${outPath}/${circuit.name}_cpp && make`); + // move the binary and dat out + fs.renameSync(`${outPath}/${circuit.name}_cpp/${circuit.name}`, `${outPath}/${circuit.name}`); + fs.renameSync(`${outPath}/${circuit.name}_cpp/${circuit.name}.dat`, `${outPath}/${circuit.name}.dat`); + } catch (error) { + throw new Error(`Failed to compile the c witness for ${circuit.name}`); + } + } else { + // move the wasm out + fs.renameSync(`${outPath}/${circuit.name}_js/${circuit.name}.wasm`, `${outPath}/${circuit.name}.wasm`); + } + + // read the content of the directory and move the files to the output folder + // don't do it recursively because we would have moved any useful files + // inside the folder already from the steps above + if (outputPathUpdated) { + const files = fs.readdirSync(outPath, { recursive: false }); + files.forEach((file) => { + const fileStr = file.toString(); + if (fs.statSync(`${outPath}/${fileStr}`).isFile() && path.extname(`${outPath}/${fileStr}`) !== ".r1cs") { + fs.renameSync(`${outPath}/${fileStr}`, `${outputPathUpdated}/${fileStr}`); + } + }); + } + } + + return circomKitConfig.dirBuild; +}; + +if (require.main === module) { + (async () => { + const cWitness = process.argv.includes("--cWitness"); + const outputPathIndex = process.argv.indexOf("--outPath"); + if (outputPathIndex === -1) { + await compileCircuits(cWitness); + } else { + const outputFolder = process.argv[outputPathIndex + 1]; + await compileCircuits(cWitness, outputFolder); + } + })(); +} diff --git a/circuits/ts/genZkeys.ts b/circuits/ts/genZkeys.ts new file mode 100644 index 0000000000..f9ea84648e --- /dev/null +++ b/circuits/ts/genZkeys.ts @@ -0,0 +1,69 @@ +import { type CircomkitConfig, type CircuitConfig, Circomkit } from "circomkit"; + +import fs from "fs"; +import path from "path"; + +import type { CircuitConfigWithName } from "./types"; + +import { cleanThreads } from "./utils"; + +/** + * Generate the zkeys for MACI's circuits using circomkit + * @param outPath - the path to the output folder + */ +export const generateZkeys = async (outputPath?: string): Promise => { + // read circomkit config files + const configFilePath = path.resolve(__dirname, "..", "circomkit.json"); + const circomKitConfig = JSON.parse(fs.readFileSync(configFilePath, "utf-8")) as CircomkitConfig; + const circuitsConfigPath = path.resolve(__dirname, "..", "circom", "circuits.json"); + const circuitsConfigContent = JSON.parse(fs.readFileSync(circuitsConfigPath, "utf-8")) as unknown as Record< + string, + CircuitConfig + >; + const circuitsConfigs: CircuitConfigWithName[] = Object.entries(circuitsConfigContent).map(([name, config]) => ({ + name, + ...config, + })); + + const outPath = outputPath ? path.resolve(outputPath) : undefined; + // update the output directory + if (outPath) { + circomKitConfig.dirBuild = outPath; + circomKitConfig.dirPtau = outPath; + } + + const circomkitInstance = new Circomkit({ + ...circomKitConfig, + verbose: false, + }); + + // loop through each circuit config and compile them + // eslint-disable-next-line @typescript-eslint/prefer-for-of + for (let i = 0; i < circuitsConfigs.length; i += 1) { + const circuit = circuitsConfigs[i]; + + // eslint-disable-next-line no-await-in-loop + const { proverKeyPath } = await circomkitInstance.setup(circuit.name); + // rename to zkey as circomkit changes to pkey and move to the output folder + if (outPath) { + fs.renameSync(proverKeyPath, path.resolve(outPath, `${circuit.name}.0.zkey`)); + } else { + fs.renameSync(proverKeyPath, `${proverKeyPath}.0.zkey`); + } + } + + // clean up the threads so we can exit + await cleanThreads(); +}; + +if (require.main === module) { + (async () => { + const outputPathIndex = process.argv.indexOf("--outPath"); + if (outputPathIndex === -1) { + await generateZkeys(); + } else { + const outputFolder = process.argv[process.argv.indexOf("--outPath") + 1]; + await generateZkeys(outputFolder); + } + })(); +} diff --git a/circuits/ts/types.ts b/circuits/ts/types.ts index 161ec353fb..aaa78b21ad 100644 --- a/circuits/ts/types.ts +++ b/circuits/ts/types.ts @@ -1,3 +1,4 @@ +import type { CircuitConfig } from "circomkit"; import type { CircuitInputs } from "maci-core"; /** @@ -65,3 +66,10 @@ export interface ITallyVotesInputs { newPerVOSpentVoiceCreditsRootSalt: bigint; newSpentVoiceCreditSubtotalSalt: bigint; } + +/** + * Extend CircuitConfig type to include the name of the circuit + */ +export interface CircuitConfigWithName extends CircuitConfig { + name: string; +} diff --git a/cli/package.json b/cli/package.json index 57982819f8..937b4d2bfc 100644 --- a/cli/package.json +++ b/cli/package.json @@ -12,7 +12,7 @@ "scripts": { "watch": "tsc --watch", "build": "tsc -p tsconfig.build.json", - "postbuild": "cp package.json ./build", + "postbuild": "cp package.json ./build && mkdir -p ./zkeys", "types": "tsc -p tsconfig.json --noEmit", "test": "nyc ts-mocha --exit tests/**/*.test.ts", "test:e2e": "ts-mocha --exit tests/e2e/e2e.test.ts", @@ -53,8 +53,7 @@ "nyc": "^15.1.0", "snarkjs": "^0.7.2", "ts-mocha": "^10.0.0", - "typescript": "^5.3.3", - "zkey-manager": "^0.1.2" + "typescript": "^5.3.3" }, "nyc": { "reporter": [ diff --git a/cli/zkeys.config.yml b/cli/zkeys.config.yml deleted file mode 100644 index cfa75e8d0c..0000000000 --- a/cli/zkeys.config.yml +++ /dev/null @@ -1,150 +0,0 @@ ---- -out: "./zkeys" - -# Make sure that these paths are correct -circomPath: "../../.local/bin/circom" -snarkjsPath: "./node_modules/snarkjs/build/cli.cjs" - -circuits: - - template: "../circuits/circom/processMessages.circom" - component: "ProcessMessages" - params: [10, 2, 1, 2] # stateTreeDepth, msgTreeDepth, msgBatchDepth, voteOptionTreeDepth - type: "test" - pubInputs: ["inputHash"] - - #- template: "../circuits/circom/processMessages.circom" - #component: "ProcessMessages" - #params: [10, 8, 2, 6] - #type: "prod" - #pubInputs: ["inputHash"] - - - template: "../circuits/circom/tallyVotes.circom" - component: "TallyVotes" - params: [10, 1, 2] # stateTreeDepth, intStateTreeDepth, voteOptionTreeDepth - type: "test" - pubInputs: ["inputHash"] - - #- template: "../circuits/circom/tallyVotes.circom" - #component: "TallyVotes" - #params: [10, 3, 6] - #type: "prod" - #pubInputs: ["inputHash"] - - - template: "../circuits/circom/subsidy.circom" - component: "SubsidyPerBatch" - params: [10, 1, 2] # stateTreeDepth, intStateTreeDepth, voteOptionTreeDepth - type: "test" - pubInputs: ["inputHash"] - -ptauFiles: - 1: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_01.ptau" - name: "powersOfTau28_hez_final_01.ptau" - - 2: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_02.ptau" - name: "powersOfTau28_hez_final_02.ptau" - - 3: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_03.ptau" - name: "powersOfTau28_hez_final_03.ptau" - - 4: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_04.ptau" - name: "powersOfTau28_hez_final_04.ptau" - - 5: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_05.ptau" - name: "powersOfTau28_hez_final_05.ptau" - - 6: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_06.ptau" - name: "powersOfTau28_hez_final_06.ptau" - - 7: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_07.ptau" - name: "powersOfTau28_hez_final_7.ptau" - - 8: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_08.ptau" - name: "powersOfTau28_hez_final_8.ptau" - - 9: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_09.ptau" - name: "powersOfTau28_hez_final_9.ptau" - - 10: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_10.ptau" - name: "powersOfTau28_hez_final_10.ptau" - - 11: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_11.ptau" - name: "powersOfTau28_hez_final_11.ptau" - - 12: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_12.ptau" - name: "powersOfTau28_hez_final_12.ptau" - - 13: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_13.ptau" - name: "powersOfTau28_hez_final_13.ptau" - - 14: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_14.ptau" - name: "powersOfTau28_hez_final_14.ptau" - - 15: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_15.ptau" - name: "powersOfTau28_hez_final_15.ptau" - - 16: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_16.ptau" - name: "powersOfTau28_hez_final_16.ptau" - - 17: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_17.ptau" - name: "powersOfTau28_hez_final_17.ptau" - - 18: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_18.ptau" - name: "powersOfTau28_hez_final_18.ptau" - - 19: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_19.ptau" - name: "powersOfTau28_hez_final_19.ptau" - - 20: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_20.ptau" - name: "powersOfTau28_hez_final_20.ptau" - - 21: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_21.ptau" - name: "powersOfTau28_hez_final_21.ptau" - - 22: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_22.ptau" - name: "powersOfTau28_hez_final_22.ptau" - - 23: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_23.ptau" - name: "powersOfTau28_hez_final_23.ptau" - - 24: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_24.ptau" - name: "powersOfTau28_hez_final_24.ptau" - - 25: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_25.ptau" - name: "powersOfTau28_hez_final_25.ptau" - - 26: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_26.ptau" - name: "powersOfTau28_hez_final_26.ptau" - - 27: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final_27.ptau" - name: "powersOfTau28_hez_final_27.ptau" - - 28: - url: "https://storage.googleapis.com/zkevm/ptau/powersOfTau28_hez_final.ptau" - name: "powersOfTau28_hez_final_28.ptau" diff --git a/contracts/package.json b/contracts/package.json index 57ffa0fae1..9aa832a49b 100644 --- a/contracts/package.json +++ b/contracts/package.json @@ -48,6 +48,7 @@ "@types/mocha": "^10.0.6", "@types/node": "^20.11.0", "@types/snarkjs": "^0.7.7", + "big-integer": "^1.6.52", "chai": "^4.3.10", "hardhat-artifactor": "^0.2.0", "hardhat-contract-sizer": "^2.10.0", diff --git a/package.json b/package.json index 3003034ee7..8df8f667b5 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,9 @@ "license": "MIT", "scripts": { "build": "lerna run build --ignore=website", + "build:circuits-c": "lerna run build-test-circuits-c --outPath ../cli/zkeys --scope \"maci-circuits\"", + "build:circuits-wasm": "lerna run build-test-circuits-wasm --outPath ../cli/zkeys --scope \"maci-circuits\"", + "setup:zkeys": "NODE_OPTIONS=--max-old-space-size=4096 lerna run gen-zkeys -- --scope \"maci-circuits\"", "clean": "lerna exec -- rm -rf node_modules build && rm -rf node_modules", "commit": "git cz", "prettier": "prettier -c .", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e380d07dec..abe6dd670f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -137,6 +137,9 @@ importers: ts-mocha: specifier: ^10.0.0 version: 10.0.0(mocha@10.2.0) + ts-node: + specifier: ^10.9.1 + version: 10.9.2(@types/node@20.11.0)(typescript@5.3.3) typescript: specifier: ^5.3.3 version: 5.3.3 @@ -219,9 +222,6 @@ importers: typescript: specifier: ^5.3.3 version: 5.3.3 - zkey-manager: - specifier: ^0.1.2 - version: 0.1.2 contracts: dependencies: @@ -274,6 +274,9 @@ importers: "@types/snarkjs": specifier: ^0.7.7 version: 0.7.7 + big-integer: + specifier: ^1.6.52 + version: 1.6.52 chai: specifier: ^4.3.10 version: 4.4.0 @@ -3676,14 +3679,6 @@ packages: fastfile: 0.0.20 ffjavascript: 0.2.62 - /@iden3/binfileutils@0.0.8: - resolution: - { integrity: sha512-/GqTsujUssGuQY+sd/XaLrA+OiCwzm+6yH28C57QQDWCHET2Logry9fGxU10n6XKdhCQBjZ7T/YMQkLwwkpRTQ== } - dependencies: - fastfile: 0.0.19 - ffjavascript: 0.2.37 - dev: true - /@isaacs/cliui@8.0.2: resolution: { integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== } @@ -7711,15 +7706,6 @@ packages: inherits: 2.0.4 safe-buffer: 5.2.1 - /circom_runtime@0.1.13: - resolution: - { integrity: sha512-vmv19/0p5OTe5uCI7PWqPtB5vPoYWjczqKYnabaC5HOxX99R4K1MuNqEXsNEAoEfZrmfAQd7vXLcATN9NVnsPA== } - hasBin: true - dependencies: - ffjavascript: 0.2.35 - fnv-plus: 1.3.1 - dev: true - /circom_runtime@0.1.21: resolution: { integrity: sha512-qTkud630B/GK8y76hnOaaS1aNuF6prfV0dTrkeRsiJKnlP1ryQbP2FWLgDOPqn6aKyaPlam+Z+DTbBhkEzh8dA== } @@ -7764,6 +7750,7 @@ packages: /circomlib@2.0.5: resolution: { integrity: sha512-O7NQ8OS+J4eshBuoy36z/TwQU0YHw8W3zxZcs4hVwpEll3e4hDm3mgkIPqItN8FDeLEKZFK3YeT/+k8TiLF3/A== } + dev: false /circomlibjs@0.0.8: resolution: @@ -10483,11 +10470,6 @@ packages: punycode: 1.4.1 dev: false - /fastfile@0.0.19: - resolution: - { integrity: sha512-tz9nWR5KYb6eR2odFQ7oxqEkx8F3YQZ6NBJoJR92YEG3DqYOqyxMck8PKvTVNKx3uwvOqGnLXNScnqpdHRdHGQ== } - dev: true - /fastfile@0.0.20: resolution: { integrity: sha512-r5ZDbgImvVWCP0lA/cGNgQcZqR+aYdFx3u+CtJqUE510pBUVGMn4ulL/iRTI4tACTYsNJ736uzFxEBXesPAktA== } @@ -10521,24 +10503,6 @@ packages: xml-js: 1.6.11 dev: false - /ffjavascript@0.2.35: - resolution: - { integrity: sha512-xnC51tWbi0ah4SH+02jEfJyO+P+NiZWnxQrLDLtBYY1Dv3QM5ydxzd+gxnLEfWdT8i1bMM5pIh5P25l6fNCaVQ== } - dependencies: - big-integer: 1.6.52 - wasmcurves: 0.0.14 - web-worker: 1.3.0 - dev: true - - /ffjavascript@0.2.37: - resolution: - { integrity: sha512-wWOkQB07d6eKFRMMtryTVsWrMsR4ImdikhEuu6ZcFYaxufJaBKUiQHYxsPzZEScPa+okkHzf1/2YVPU7xUIHMQ== } - dependencies: - big-integer: 1.6.52 - wasmcurves: 0.0.14 - web-worker: 1.3.0 - dev: true - /ffjavascript@0.2.56: resolution: { integrity: sha512-em6G5Lrj7ucIqj4TYEgyoHs/j99Urwwqa4+YxEVY2hggnpRimVj+noX5pZQTxI1pvtiekZI4rG65JBf0xraXrg== } @@ -10739,6 +10703,7 @@ packages: /fnv-plus@1.3.1: resolution: { integrity: sha512-Gz1EvfOneuFfk4yG458dJ3TLJ7gV19q3OM/vVvvHf7eT02Hm1DleB4edsia6ahbKgAYxO9gvyQ1ioWZR+a00Yw== } + dev: false /follow-redirects@1.15.4(debug@4.3.4): resolution: @@ -12560,6 +12525,7 @@ packages: resolution: { integrity: sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== } engines: { node: ">= 0.10" } + dev: false /invariant@2.2.4: resolution: @@ -13251,14 +13217,6 @@ packages: argparse: 1.0.10 esprima: 4.0.1 - /js-yaml@4.0.0: - resolution: - { integrity: sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q== } - hasBin: true - dependencies: - argparse: 2.0.1 - dev: true - /js-yaml@4.1.0: resolution: { integrity: sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA== } @@ -17315,16 +17273,6 @@ packages: dev: false optional: true - /r1csfile@0.0.32: - resolution: - { integrity: sha512-DkRXeOg0iRmfhgIuWICvdkOiLHpyb7+AcUd/WHpqBJEUp27pe7wKXBR4Jr3TPYCT4sTV9a/F3bovyAC4wystnQ== } - dependencies: - "@iden3/bigarray": 0.0.2 - "@iden3/binfileutils": 0.0.8 - fastfile: 0.0.19 - ffjavascript: 0.2.35 - dev: true - /r1csfile@0.0.41: resolution: { integrity: sha512-Q1WDF3u1vYeAwjHo4YuddkA8Aq0TulbKjmGm99+Atn13Lf5fTsMZBnBV9T741w8iSyPFG6Uh6sapQby77sREqA== } @@ -17694,17 +17642,13 @@ packages: { integrity: sha512-onYyVhBNr4CmAxFsKS7bz+uTLRakypIe4R+5A824vBSkQy/hB3fZepoVEf8OVAxzLvK+H/jm9TzpI3ETSm64Kg== } dev: false - /readline@1.3.0: - resolution: - { integrity: sha512-k2d6ACCkiNYz222Fs/iNze30rRJ1iIicW7JuX/7/cozvih6YCkFZH+J6mAFDVgv0dRBaAyr4jDqC95R2y4IADg== } - dev: true - /rechoir@0.6.2: resolution: { integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw== } engines: { node: ">= 0.10" } dependencies: resolve: 1.22.8 + dev: false /recursive-readdir@2.2.3: resolution: @@ -18606,6 +18550,7 @@ packages: glob: 7.2.3 interpret: 1.4.0 rechoir: 0.6.2 + dev: false /shiki@0.14.7: resolution: @@ -18756,24 +18701,6 @@ packages: engines: { node: ">= 6.0.0", npm: ">= 3.0.0" } dev: true - /snarkjs@0.4.7: - resolution: - { integrity: sha512-gehpEKZwhBZEYPLokfBqtHIxaHHy50LYDI4xFVopNEwICInHPIbG4nJ0nJ1vgqz+bv824UFQC84XnQ0C/mPjag== } - deprecated: Sucurity bug fixed - hasBin: true - dependencies: - "@iden3/binfileutils": 0.0.8 - blake2b-wasm: 2.4.0 - circom_runtime: 0.1.13 - ejs: 3.1.9 - fastfile: 0.0.19 - ffjavascript: 0.2.37 - js-sha3: 0.8.0 - logplease: 1.2.15 - r1csfile: 0.0.32 - readline: 1.3.0 - dev: true - /snarkjs@0.5.0: resolution: { integrity: sha512-KWz8mZ2Y+6wvn6GGkQo6/ZlKwETdAGohd40Lzpwp5TUZCn6N6O4Az1SuX1rw/qREGL6Im+ycb19suCFE8/xaKA== } @@ -20604,14 +20531,6 @@ packages: resolution: { integrity: sha512-Qx3lEFqaVvp1cEYW7Bfi+ebRJrOiwz2Ieu7ZG2l7YyeSJIok/reEQCQCuicj/Y32ITIJuGIM9xZQppGx5LrQdA== } - /wasmcurves@0.0.14: - resolution: - { integrity: sha512-G1iMkxlRaQSdqQ1JrwHcU+awLmwyH6kFKfT8g9obd8MWe+u5oSdFXrODB0zmSI5aGGvJPG+4cAmqCGYv9R+7qg== } - dependencies: - big-integer: 1.6.52 - blakejs: 1.2.1 - dev: true - /wasmcurves@0.2.0: resolution: { integrity: sha512-3e2rbxdujOwaod657gxgmdhZNn+i1qKdHO3Y/bK+8E7bV8ttV/fu5FO4/WLBACF375cK0QDLOP+65Na63qYuWA== } @@ -21635,18 +21554,6 @@ packages: engines: { node: ">=12.20" } dev: false - /zkey-manager@0.1.2: - resolution: - { integrity: sha512-Ig+QQrS8IZYkLgbdrbdCgji+1NeEatzWoIqH1G0OoF9+cNrf/qFOFr1hq2k52wVcUudCboqGKDpPbl3KRpnpVw== } - hasBin: true - dependencies: - argparse: 2.0.1 - circomlib: 2.0.5 - js-yaml: 4.0.0 - shelljs: 0.8.5 - snarkjs: 0.4.7 - dev: true - /zwitch@2.0.4: resolution: { integrity: sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A== } diff --git a/website/versioned_docs/version-v1.x/circuits.md b/website/versioned_docs/version-v1.x/circuits.md index 0b5c2f2ed0..bfef1fefaa 100644 --- a/website/versioned_docs/version-v1.x/circuits.md +++ b/website/versioned_docs/version-v1.x/circuits.md @@ -427,24 +427,30 @@ Before building the project, make sure you have the following dependencies insta To build the two main circuits of MACI, run the following commands: ``` -circom --r1cs --sym --wasm --output ./build circom/test/processMessages_test.circom -circom --r1cs --sym --wasm --output ./build circom/test/tallyVotes_test.circom +circom --r1cs --sym --wasm --output ./build circom/test/processMessages_10-2-1-2_test.circom +circom --r1cs --sym --wasm --output ./build circom/test/tallyVotes_10-1-2_test.circom ``` Please note that the circuit is configured with testing purpose parameters, which means it can only handle a limited amount of messages (up to 25 messages). For more information on the parameters and how to configure them, refer to [this page](/docs/circuits/#compile-circuits). ### Generating zKeys -Run: +Run from the root directory to save to the `cli/zkeys`` folder: ```bash -pnpm exec zkey-manager compile -c ./zkeys.config.yml +pnpm setup:zkeys +``` + +Run from the circuits folder with `--outPath` to save to a custom folder: + +```bash +cd circuits && pnpm gen-zkeys --outPath ./customFolder ``` The larger the trees, the more time this process may take. You may also need a machine with a very large amount of memory. -> Note that you will have to modify the parameters inside the `zkeys.config.yml` file to match your use case. For example, if you want to support up to 3125 messages, the message tree depth parameter should be set to `5` (as $5^5 = 3125$). +> Note that you will have to modify the parameters inside the `./circuits/circom/circuits.json` file to match your use case. For example, if you want to support up to 3125 messages, the message tree depth parameter should be set to `5` (as $5^5 = 3125$). #### Measure the circuit sizes @@ -464,14 +470,7 @@ This file should be the result of the Perpetual Powers of Tau trusted setup contribution which [Hermez Network selected](https://blog.hermez.io/hermez-cryptographic-setup/). -Run: - -```bash -pnpm exec zkey-manager downloadPtau -c ./zkeys.config.yml -``` - -`zkey-manager` will select the smallest `.ptau` file that fits the largest -circuit specified in `zkeys.config.yml`. +When running the `setup:zkeys` command, the `.ptau` file will be downloaded automatically. ### Generating and Validating ZK Proofs diff --git a/website/versioned_docs/version-v1.x/installation.md b/website/versioned_docs/version-v1.x/installation.md index 6faba64cd6..67f650cede 100644 --- a/website/versioned_docs/version-v1.x/installation.md +++ b/website/versioned_docs/version-v1.x/installation.md @@ -61,7 +61,7 @@ pnpm run build #### On intel chips -Install dependencies for and `zkey-manager`: +Install dependencies to compile the c++ witness generator: ```bash sudo apt-get install libgmp-dev nlohmann-json3-dev nasm g++ @@ -69,17 +69,34 @@ sudo apt-get install libgmp-dev nlohmann-json3-dev nasm g++ > Note that on an ARM macbook you won't need the above. However, you will not be able to compile the c++ witness generator. -### Configure zkey-manager - -Edit `cli/zkeys.config.yml` to include the relative path to the -circom binary. - -```yml ---- -circomPath: "RELATIVE_PATH_TO_CIRCOM" +### Configure circomkit + +Edit `circuits/circom/circuits` to include the circuits you would like to compile. This comes already configured with the three main circuits and with testing parameters: + +```json +{ + "ProcessMessages_10-2-1-2_test": { + "file": "processMessages", + "template": "ProcessMessages", + "params": [10, 2, 1, 2], + "pubs": ["inputHash"] + }, + "TallyVotes_10-1-2_test": { + "file": "tallyVotes", + "template": "TallyVotes", + "params": [10, 1, 2], + "pubs": ["inputHash"] + }, + "SubsidyPerBatch_10-1-2_test": { + "file": "subsidy", + "template": "SubsidyPerBatch", + "params": [10, 1, 2], + "pubs": ["inputHash"] + } +} ``` -### Download `.zkey` files +### Download `.zkey` files (if you would like to use the default parameters or the trusted setup artifacts) MACI has two main zk-SNARK circuits (plus an optional Subsidy circuit). Each circuit is parameterised. There should one `.zkey` file for each circuit and set of parameters. @@ -92,30 +109,32 @@ Note the locations of the `.zkey` files as the CLI requires them as command-line ### Generate `.zkey` files -If you wish to generate `.zkey` files from scratch, first navigate to `cli/` -and edit `zkeys.config.yml`. Set the parameters you need. +If you wish to generate `.zkey` files from scratch, first navigate to `circuits/circom` +and edit `circuits.json`. Set the parameters you need. Next, run the following to compile the circuits with parameters you specified: +**for the c++ witness generator** + ```bash -pnpm exec zkey-manager compile -c zkeys.config.yml +pnpm test:circuits-c ``` -Next, download the `.ptau` file: +\*_for the wasm witness generator_ ```bash -pnpm exec zkey-manager downloadPtau -c zkeys.config.yml +pnpm build:circuits-wasm ``` Finally, generate the `.zkey` files. This may require a lot of memory and time. ```bash -pnpm exec zkey-manager genZkeys -c zkeys.config.yml +pnpm setup:test-zkeys ``` -> If on a ARM64 chip, the above will work, though it will show errors for the c++ witness generator. You can ignore these errors. +> If on a ARM64 chip, the above will work with the wasm witness only. The errors you will get for the c++ witness are: > -> ``` +> ```bash > main.cpp:9:10: fatal error: 'nlohmann/json.hpp' file not found > #include > ^~~~~~~~~~~~~~~~~~~ diff --git a/website/versioned_docs/version-v1.x/testing.md b/website/versioned_docs/version-v1.x/testing.md index 35e1afb448..1049cf3b85 100644 --- a/website/versioned_docs/version-v1.x/testing.md +++ b/website/versioned_docs/version-v1.x/testing.md @@ -88,34 +88,39 @@ You can run the script directly with bash or use pnpm: `pnpm run download-zkeys` ### Compile the witness generation binaries (if generating from scratch) -From the main `maci/cli` directory, run: +From the root folder, run: +**for c++ witness generator** + +```bash +pnpm build:circuits-c ``` -pnpm exec zkey-manager compile -c ./zkeys.config.yml + +**for wasm witness generator** + +```bash +pnpm build:circuits-wasm ``` You should see the following files in `maci/cli/zkeys/`: -``` +```bash ProcessMessages_10-2-1-2_test -ProcessMessages_10-2-1-2_test.circom ProcessMessages_10-2-1-2_test.dat -ProcessMessages_10-2-1-2_test.r1cs ProcessMessages_10-2-1-2_test.sym +# either cpp or js ProcessMessages_10-2-1-2_test_cpp ProcessMessages_10-2-1-2_test_js SubsidyPerBatch_10-1-2_test -SubsidyPerBatch_10-1-2_test.circom SubsidyPerBatch_10-1-2_test.dat -SubsidyPerBatch_10-1-2_test.r1cs SubsidyPerBatch_10-1-2_test.sym +# either cpp or js SubsidyPerBatch_10-1-2_test_cpp SubsidyPerBatch_10-1-2_test_js TallyVotes_10-1-2_test -TallyVotes_10-1-2_test.circom TallyVotes_10-1-2_test.dat -TallyVotes_10-1-2_test.r1cs TallyVotes_10-1-2_test.sym +# either cpp or js TallyVotes_10-1-2_test_cpp TallyVotes_10-1-2_test_js ``` diff --git a/website/versioned_docs/version-v1.x/troubleshooting.md b/website/versioned_docs/version-v1.x/troubleshooting.md index 0eb4e23aae..f759a4aa93 100644 --- a/website/versioned_docs/version-v1.x/troubleshooting.md +++ b/website/versioned_docs/version-v1.x/troubleshooting.md @@ -47,7 +47,6 @@ Error: Error executing ./zkeys/ProcessMessages_10-2-1-2_test /tmp/tmp-9904-zG0k8 You can generate the missing `.dat` files using the following command: -``` -cd cli/ -pnpm exec zkey-manager compile -c zkeys.config.yml +```bash +pnpm build:circuits-c ```