From 43a651ac767d8ca355d2cf7d36b275e3a5d85324 Mon Sep 17 00:00:00 2001 From: unstark Date: Wed, 19 Jun 2024 21:22:05 +0100 Subject: [PATCH] Prover service trait, SHARP client, fact checker, Stone stubs (#6) * Prover service trait and SHARP client implementation * _ to - in crates * fix taplo * fix taplo again --------- Co-authored-by: apoorvsadana <95699312+apoorvsadana@users.noreply.github.com> --- .github/.DS_Store | Bin 6148 -> 0 bytes Cargo.lock | 4351 ++++++++++++++--- Cargo.toml | 74 +- .../da-client-interface/Cargo.toml | 2 +- .../da-client-interface/src/lib.rs | 3 +- .../ethereum/.env_example | 0 .../ethereum/Cargo.toml | 0 .../ethereum/src/config.rs | 0 .../ethereum/src/lib.rs | 24 +- .../ethereum/test_utils/hex_block_630872.txt | 0 .../ethereum/trusted_setup.txt | 0 crates/orchestrator/Cargo.toml | 4 + crates/orchestrator/src/config.rs | 46 +- .../src/controllers/jobs_controller.rs | 8 +- crates/orchestrator/src/database/mod.rs | 6 +- .../src/database/mongodb/config.rs | 3 +- .../orchestrator/src/database/mongodb/mod.rs | 29 +- crates/orchestrator/src/jobs/constants.rs | 1 + crates/orchestrator/src/jobs/da_job/mod.rs | 41 +- crates/orchestrator/src/jobs/mod.rs | 33 +- .../orchestrator/src/jobs/prover_job/mod.rs | 78 + .../src/jobs/register_proof_job/mod.rs | 10 +- crates/orchestrator/src/jobs/snos_job/mod.rs | 10 +- .../src/jobs/state_update_job/mod.rs | 10 +- crates/orchestrator/src/jobs/types.rs | 8 +- crates/orchestrator/src/lib.rs | 2 - crates/orchestrator/src/main.rs | 2 +- crates/orchestrator/src/queue/job_queue.rs | 10 +- crates/orchestrator/src/queue/mod.rs | 6 +- crates/orchestrator/src/queue/sqs/mod.rs | 6 +- crates/orchestrator/src/routes.rs | 3 +- .../src/tests/artifacts/fibonacci.zip | Bin 0 -> 1514 bytes crates/orchestrator/src/tests/common/mod.rs | 25 +- .../orchestrator/src/tests/jobs/da_job/mod.rs | 31 +- crates/orchestrator/src/tests/jobs/mod.rs | 3 + .../src/tests/jobs/prover_job/mod.rs | 78 + crates/orchestrator/src/tests/server/mod.rs | 15 +- crates/orchestrator/src/tests/workers/mod.rs | 21 +- crates/orchestrator/src/utils/env_utils.rs | 13 - crates/orchestrator/src/utils/mod.rs | 1 - crates/orchestrator/src/workers/mod.rs | 3 +- .../src/workers/proof_registration.rs | 6 +- crates/orchestrator/src/workers/proving.rs | 6 +- crates/orchestrator/src/workers/snos.rs | 10 +- .../orchestrator/src/workers/update_state.rs | 6 +- .../gps-fact-checker/Cargo.toml | 22 + .../gps-fact-checker/src/error.rs | 53 + .../gps-fact-checker/src/fact_info.rs | 98 + .../gps-fact-checker/src/fact_node.rs | 117 + .../gps-fact-checker/src/fact_topology.rs | 102 + .../gps-fact-checker/src/lib.rs | 40 + .../tests/artifacts/FactRegistry.json | 863 ++++ .../tests/artifacts/README.md | 39 + .../tests/artifacts/fibonacci.zip | Bin 0 -> 1514 bytes .../tests/artifacts/get_fact.py | 14 + .../prover-client-interface/Cargo.toml | 13 + .../prover-client-interface/src/lib.rs | 45 + .../prover-services/sharp-service/Cargo.toml | 25 + .../sharp-service/src/client.rs | 49 + .../sharp-service/src/config.rs | 27 + .../sharp-service/src/error.rs | 30 + .../prover-services/sharp-service/src/lib.rs | 140 + .../tests/artifacts/fibonacci.zip | Bin 0 -> 1514 bytes .../settlement-client-interface/Cargo.toml | 0 .../settlement-client-interface/src/lib.rs | 3 +- crates/utils/Cargo.toml | 2 + crates/utils/src/lib.rs | 13 + crates/utils/src/settings/default.rs | 13 + crates/utils/src/settings/mod.rs | 13 + e2e-tests/Cargo.toml | 18 + e2e-tests/src/lib.rs | 28 + e2e-tests/src/mongodb.rs | 34 + e2e-tests/src/node.rs | 89 + e2e-tests/test_samples.rs | 15 + migrations/.DS_Store | Bin 6148 -> 0 bytes 75 files changed, 6024 insertions(+), 869 deletions(-) delete mode 100644 .github/.DS_Store rename crates/{da_clients => da-clients}/da-client-interface/Cargo.toml (91%) rename crates/{da_clients => da-clients}/da-client-interface/src/lib.rs (95%) rename crates/{da_clients => da-clients}/ethereum/.env_example (100%) rename crates/{da_clients => da-clients}/ethereum/Cargo.toml (100%) rename crates/{da_clients => da-clients}/ethereum/src/config.rs (100%) rename crates/{da_clients => da-clients}/ethereum/src/lib.rs (97%) rename crates/{da_clients => da-clients}/ethereum/test_utils/hex_block_630872.txt (100%) rename crates/{da_clients => da-clients}/ethereum/trusted_setup.txt (100%) create mode 100644 crates/orchestrator/src/jobs/prover_job/mod.rs create mode 100644 crates/orchestrator/src/tests/artifacts/fibonacci.zip create mode 100644 crates/orchestrator/src/tests/jobs/prover_job/mod.rs delete mode 100644 crates/orchestrator/src/utils/env_utils.rs delete mode 100644 crates/orchestrator/src/utils/mod.rs create mode 100644 crates/prover-services/gps-fact-checker/Cargo.toml create mode 100644 crates/prover-services/gps-fact-checker/src/error.rs create mode 100644 crates/prover-services/gps-fact-checker/src/fact_info.rs create mode 100644 crates/prover-services/gps-fact-checker/src/fact_node.rs create mode 100644 crates/prover-services/gps-fact-checker/src/fact_topology.rs create mode 100644 crates/prover-services/gps-fact-checker/src/lib.rs create mode 100644 crates/prover-services/gps-fact-checker/tests/artifacts/FactRegistry.json create mode 100644 crates/prover-services/gps-fact-checker/tests/artifacts/README.md create mode 100644 crates/prover-services/gps-fact-checker/tests/artifacts/fibonacci.zip create mode 100644 crates/prover-services/gps-fact-checker/tests/artifacts/get_fact.py create mode 100644 crates/prover-services/prover-client-interface/Cargo.toml create mode 100644 crates/prover-services/prover-client-interface/src/lib.rs create mode 100644 crates/prover-services/sharp-service/Cargo.toml create mode 100644 crates/prover-services/sharp-service/src/client.rs create mode 100644 crates/prover-services/sharp-service/src/config.rs create mode 100644 crates/prover-services/sharp-service/src/error.rs create mode 100644 crates/prover-services/sharp-service/src/lib.rs create mode 100644 crates/prover-services/sharp-service/tests/artifacts/fibonacci.zip rename crates/{settlement_clients => settlement-clients}/settlement-client-interface/Cargo.toml (100%) rename crates/{settlement_clients => settlement-clients}/settlement-client-interface/src/lib.rs (96%) create mode 100644 crates/utils/src/settings/default.rs create mode 100644 crates/utils/src/settings/mod.rs create mode 100644 e2e-tests/Cargo.toml create mode 100644 e2e-tests/src/lib.rs create mode 100644 e2e-tests/src/mongodb.rs create mode 100644 e2e-tests/src/node.rs create mode 100644 e2e-tests/test_samples.rs delete mode 100644 migrations/.DS_Store diff --git a/.github/.DS_Store b/.github/.DS_Store deleted file mode 100644 index daf430c92957832b27a3a56edcd4965cb3f86351..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKOG*SW5UtW#w78k2%Uoe@5Qp{za{(C#73?8{gSgGYLwNz0F2ucSkKwD2GK~>j zh=^1{@+zrM(hoXG5fRUCRx_e05p`&SEJ{OUx@p>T=LwKC$2kpD=$5v_MI+H)oRYmC zV8?~7>56vtKUqzudn*Jw6hM3J6ohXG>kqN2nK?I4+i9XNN9rDu^Q^pflf;Rpgf~hpi3"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [workspace.dependencies] - - num = { version = "0.4.1" } -ethereum-da-client = { path = "crates/da_clients/ethereum" } async-trait = { version = "0.1.77" } -da-client-interface = { path = "crates/da_clients/da-client-interface" } +alloy = { git = "https://github.com/alloy-rs/alloy", rev = "7373f6db761d5a19888e3a0c527e8a3ca31e7a1e" } +alloy-primitives = "0.7.4" axum = { version = "0.7.4" } -axum-macros = { version = "0.4.1" } -color-eyre = { version = "0.6.2" } -dotenvy = { version = "0.15.7" } -futures = { version = "0.3.30" } +axum-macros = "0.4.1" +color-eyre = "0.6.2" +dotenvy = "0.15.7" +futures = "0.3.30" mongodb = { version = "2.8.1" } omniqueue = { version = "0.2.0" } -rstest = { version = "0.18.2" } +reqwest = { version = "0.11.24" } +rstest = "0.18.2" serde = { version = "1.0.197" } -serde_json = { version = "1.0.114" } -starknet = { version = "0.9.0" } -thiserror = { version = "1.0.57" } -tokio = { version = "1.36.0" } -tracing = { version = "0.1.40" } +serde_json = "1.0.114" +starknet = "0.9.0" +tempfile = "3.8.1" +thiserror = "1.0.57" +tokio = { version = "1.37.0" } +tokio-stream = "0.1.15" +tokio-util = "0.7.11" +tracing = "0.1.40" tracing-subscriber = { version = "0.3.18" } -url = { version = "2.5.0" } -uuid = { version = "1.7.0" } -num-bigint = { version = "0.4.4" } +url = { version = "2.5.0", features = ["serde"] } +uuid = { version = "1.7.0", features = ["v4", "serde"] } httpmock = { version = "0.7.0" } -utils = { path = "crates/utils" } +num-bigint = { version = "0.4.4" } arc-swap = { version = "1.7.1" } num-traits = "0.2" lazy_static = "1.4.0" +stark_evm_adapter = "0.1.1" +hex = "0.4" +itertools = "0.13.0" +mockall = "0.12.1" +testcontainers = "0.18.0" + +# Cairo VM +cairo-vm = { git = "https://github.com/lambdaclass/cairo-vm", features = [ + "extensive_hints", + "cairo-1-hints", +] } + +# Sharp (Starkware) +snos = { git = "https://github.com/keep-starknet-strange/snos" } + +# Madara prover API +madara-prover-common = { git = "https://github.com/Moonsong-Labs/madara-prover-api", branch = "od/use-latest-cairo-vm" } +madara-prover-rpc-client = { git = "https://github.com/Moonsong-Labs/madara-prover-api", branch = "od/use-latest-cairo-vm" } + +# Project +da-client-interface = { path = "crates/da-clients/da-client-interface" } +ethereum-da-client = { path = "crates/da-clients/ethereum" } +utils = { path = "crates/utils" } +prover-client-interface = { path = "crates/prover-services/prover-client-interface" } +gps-fact-checker = { path = "crates/prover-services/gps-fact-checker" } +sharp-service = { path = "crates/prover-services/sharp-service" } +orchestrator = { path = "crates/orchestrator" } diff --git a/crates/da_clients/da-client-interface/Cargo.toml b/crates/da-clients/da-client-interface/Cargo.toml similarity index 91% rename from crates/da_clients/da-client-interface/Cargo.toml rename to crates/da-clients/da-client-interface/Cargo.toml index f9dee4ca..d2887a27 100644 --- a/crates/da_clients/da-client-interface/Cargo.toml +++ b/crates/da-clients/da-client-interface/Cargo.toml @@ -9,5 +9,5 @@ edition.workspace = true async-trait = { workspace = true } axum = { workspace = true } color-eyre = { workspace = true } -mockall = "0.12.1" +mockall = { workspace = true } starknet = { workspace = true } diff --git a/crates/da_clients/da-client-interface/src/lib.rs b/crates/da-clients/da-client-interface/src/lib.rs similarity index 95% rename from crates/da_clients/da-client-interface/src/lib.rs rename to crates/da-clients/da-client-interface/src/lib.rs index 5495e8fa..5d36d2ad 100644 --- a/crates/da_clients/da-client-interface/src/lib.rs +++ b/crates/da-clients/da-client-interface/src/lib.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use color_eyre::Result; -use mockall::{automock, predicate::*}; +use mockall::automock; +use mockall::predicate::*; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub enum DaVerificationStatus { diff --git a/crates/da_clients/ethereum/.env_example b/crates/da-clients/ethereum/.env_example similarity index 100% rename from crates/da_clients/ethereum/.env_example rename to crates/da-clients/ethereum/.env_example diff --git a/crates/da_clients/ethereum/Cargo.toml b/crates/da-clients/ethereum/Cargo.toml similarity index 100% rename from crates/da_clients/ethereum/Cargo.toml rename to crates/da-clients/ethereum/Cargo.toml diff --git a/crates/da_clients/ethereum/src/config.rs b/crates/da-clients/ethereum/src/config.rs similarity index 100% rename from crates/da_clients/ethereum/src/config.rs rename to crates/da-clients/ethereum/src/config.rs diff --git a/crates/da_clients/ethereum/src/lib.rs b/crates/da-clients/ethereum/src/lib.rs similarity index 97% rename from crates/da_clients/ethereum/src/lib.rs rename to crates/da-clients/ethereum/src/lib.rs index a0cedd08..b77376a0 100644 --- a/crates/da_clients/ethereum/src/lib.rs +++ b/crates/da-clients/ethereum/src/lib.rs @@ -1,9 +1,15 @@ #![allow(missing_docs)] #![allow(clippy::missing_docs_in_private_items)] +use std::env; +use std::path::Path; +use std::str::FromStr; + use alloy::consensus::{ BlobTransactionSidecar, SignableTransaction, TxEip4844, TxEip4844Variant, TxEip4844WithSidecar, TxEnvelope, }; -use alloy::eips::{eip2718::Encodable2718, eip2930::AccessList, eip4844::BYTES_PER_BLOB}; +use alloy::eips::eip2718::Encodable2718; +use alloy::eips::eip2930::AccessList; +use alloy::eips::eip4844::BYTES_PER_BLOB; use alloy::network::{Ethereum, TxSigner}; use alloy::primitives::{bytes, Address, FixedBytes, TxHash, U256, U64}; use alloy::providers::{Provider, ProviderBuilder, RootProvider}; @@ -11,18 +17,15 @@ use alloy::rpc::client::RpcClient; use alloy::signers::wallet::LocalWallet; use alloy::transports::http::Http; use async_trait::async_trait; - -use color_eyre::Result; -use mockall::{automock, predicate::*}; -use reqwest::Client; -use std::str::FromStr; -use url::Url; - use c_kzg::{Blob, KzgCommitment, KzgProof, KzgSettings}; +use color_eyre::Result; use config::EthereumDaConfig; use da_client_interface::{DaClient, DaVerificationStatus}; use dotenv::dotenv; -use std::{env, path::Path}; +use mockall::automock; +use mockall::predicate::*; +use reqwest::Client; +use url::Url; pub mod config; pub struct EthereumDaClient { #[allow(dead_code)] @@ -143,10 +146,11 @@ async fn prepare_sidecar( #[cfg(test)] mod tests { - use super::*; use std::fs::File; use std::io::{self, BufRead}; + use super::*; + #[tokio::test] async fn test_kzg() { let trusted_setup = KzgSettings::load_trusted_setup_file(Path::new("./trusted_setup.txt")) diff --git a/crates/da_clients/ethereum/test_utils/hex_block_630872.txt b/crates/da-clients/ethereum/test_utils/hex_block_630872.txt similarity index 100% rename from crates/da_clients/ethereum/test_utils/hex_block_630872.txt rename to crates/da-clients/ethereum/test_utils/hex_block_630872.txt diff --git a/crates/da_clients/ethereum/trusted_setup.txt b/crates/da-clients/ethereum/trusted_setup.txt similarity index 100% rename from crates/da_clients/ethereum/trusted_setup.txt rename to crates/da-clients/ethereum/trusted_setup.txt diff --git a/crates/orchestrator/Cargo.toml b/crates/orchestrator/Cargo.toml index 0dd7365b..354571c7 100644 --- a/crates/orchestrator/Cargo.toml +++ b/crates/orchestrator/Cargo.toml @@ -17,6 +17,7 @@ async-std = "1.12.0" async-trait = { workspace = true } axum = { workspace = true, features = ["macros"] } axum-macros = { workspace = true } +cairo-vm = { workspace = true } color-eyre = { workspace = true } da-client-interface = { workspace = true } dotenvy = { workspace = true } @@ -31,8 +32,10 @@ num = { workspace = true } num-bigint = { workspace = true } num-traits = { workspace = true } omniqueue = { workspace = true, optional = true } +prover-client-interface = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sharp-service = { workspace = true } starknet = { workspace = true } starknet-core = "0.9.0" thiserror = { workspace = true } @@ -40,6 +43,7 @@ tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["env-filter"] } url = { workspace = true } +utils = { workspace = true } uuid = { workspace = true, features = ["v4", "serde"] } [features] diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index ef05d9eb..9d2ff112 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -1,19 +1,24 @@ -use crate::database::mongodb::config::MongoDbConfig; -use crate::database::mongodb::MongoDb; -use crate::database::{Database, DatabaseConfig}; -use crate::queue::sqs::SqsQueue; -use crate::queue::QueueProvider; -use crate::utils::env_utils::get_env_var_or_panic; +use std::sync::Arc; + use arc_swap::{ArcSwap, Guard}; -use da_client_interface::DaClient; -use da_client_interface::DaConfig; +use da_client_interface::{DaClient, DaConfig}; use dotenvy::dotenv; use ethereum_da_client::config::EthereumDaConfig; use ethereum_da_client::EthereumDaClient; +use prover_client_interface::ProverClient; +use sharp_service::SharpProverService; use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::{JsonRpcClient, Url}; -use std::sync::Arc; use tokio::sync::OnceCell; +use utils::env_utils::get_env_var_or_panic; +use utils::settings::default::DefaultSettingsProvider; +use utils::settings::SettingsProvider; + +use crate::database::mongodb::config::MongoDbConfig; +use crate::database::mongodb::MongoDb; +use crate::database::{Database, DatabaseConfig}; +use crate::queue::sqs::SqsQueue; +use crate::queue::QueueProvider; /// The app config. It can be accessed from anywhere inside the service /// by calling `config` function. @@ -22,6 +27,8 @@ pub struct Config { starknet_client: Arc>, /// The DA client to interact with the DA layer da_client: Box, + /// The service that produces proof and registers it onchain + prover_client: Box, /// The database client database: Box, /// The queue provider @@ -43,7 +50,10 @@ pub async fn init_config() -> Config { // init the queue let queue = Box::new(SqsQueue {}); - Config { starknet_client: Arc::new(provider), da_client: build_da_client(), database, queue } + let settings_provider = DefaultSettingsProvider {}; + let prover = create_prover_service(&settings_provider); + + Config { starknet_client: Arc::new(provider), da_client: build_da_client(), prover_client: prover, database, queue } } impl Config { @@ -51,10 +61,11 @@ impl Config { pub fn new( starknet_client: Arc>, da_client: Box, + prover_client: Box, database: Box, queue: Box, ) -> Self { - Self { starknet_client, da_client, database, queue } + Self { starknet_client, da_client, prover_client, database, queue } } /// Returns the starknet client @@ -67,6 +78,11 @@ impl Config { self.da_client.as_ref() } + /// Returns the proving service + pub fn prover_client(&self) -> &dyn ProverClient { + self.prover_client.as_ref() + } + /// Returns the database client pub fn database(&self) -> &dyn Database { self.database.as_ref() @@ -115,3 +131,11 @@ fn build_da_client() -> Box { _ => panic!("Unsupported DA layer"), } } + +/// Creates prover service based on the environment variable PROVER_SERVICE +fn create_prover_service(settings_provider: &impl SettingsProvider) -> Box { + match get_env_var_or_panic("PROVER_SERVICE").as_str() { + "sharp" => Box::new(SharpProverService::with_settings(settings_provider)), + _ => panic!("Unsupported prover service"), + } +} diff --git a/crates/orchestrator/src/controllers/jobs_controller.rs b/crates/orchestrator/src/controllers/jobs_controller.rs index 4ac8c388..43a67652 100644 --- a/crates/orchestrator/src/controllers/jobs_controller.rs +++ b/crates/orchestrator/src/controllers/jobs_controller.rs @@ -1,8 +1,10 @@ -use crate::controllers::errors::AppError; -use crate::jobs::types::JobType; +use std::collections::HashMap; + use axum::extract::Json; use serde::Deserialize; -use std::collections::HashMap; + +use crate::controllers::errors::AppError; +use crate::jobs::types::JobType; /// Client request to create a job #[derive(Debug, Deserialize)] diff --git a/crates/orchestrator/src/database/mod.rs b/crates/orchestrator/src/database/mod.rs index 3da862af..e5d5f1f7 100644 --- a/crates/orchestrator/src/database/mod.rs +++ b/crates/orchestrator/src/database/mod.rs @@ -1,10 +1,12 @@ -use crate::jobs::types::{JobItem, JobStatus, JobType}; +use std::collections::HashMap; + use async_trait::async_trait; use color_eyre::Result; use mockall::automock; -use std::collections::HashMap; use uuid::Uuid; +use crate::jobs::types::{JobItem, JobStatus, JobType}; + /// MongoDB pub mod mongodb; diff --git a/crates/orchestrator/src/database/mongodb/config.rs b/crates/orchestrator/src/database/mongodb/config.rs index aea02a43..6ec561da 100644 --- a/crates/orchestrator/src/database/mongodb/config.rs +++ b/crates/orchestrator/src/database/mongodb/config.rs @@ -1,5 +1,6 @@ +use utils::env_utils::get_env_var_or_panic; + use crate::database::DatabaseConfig; -use crate::utils::env_utils::get_env_var_or_panic; pub struct MongoDbConfig { pub url: String, diff --git a/crates/orchestrator/src/database/mongodb/mod.rs b/crates/orchestrator/src/database/mongodb/mod.rs index 308fb43e..95a4b763 100644 --- a/crates/orchestrator/src/database/mongodb/mod.rs +++ b/crates/orchestrator/src/database/mongodb/mod.rs @@ -1,19 +1,17 @@ -use crate::database::mongodb::config::MongoDbConfig; -use crate::database::Database; -use crate::jobs::types::{JobItem, JobStatus, JobType}; +use std::collections::HashMap; + use async_trait::async_trait; use color_eyre::eyre::eyre; use color_eyre::Result; -use mongodb::bson::Document; -use mongodb::options::{FindOneOptions, UpdateOptions}; -use mongodb::{ - bson::doc, - options::{ClientOptions, ServerApi, ServerApiVersion}, - Client, Collection, -}; -use std::collections::HashMap; +use mongodb::bson::{doc, Document}; +use mongodb::options::{ClientOptions, FindOneOptions, ServerApi, ServerApiVersion, UpdateOptions}; +use mongodb::{Client, Collection}; use uuid::Uuid; +use crate::database::mongodb::config::MongoDbConfig; +use crate::database::Database; +use crate::jobs::types::{JobItem, JobStatus, JobType}; + pub mod config; pub struct MongoDb { @@ -23,7 +21,8 @@ pub struct MongoDb { impl MongoDb { pub async fn new(config: MongoDbConfig) -> Self { let mut client_options = ClientOptions::parse(config.url).await.expect("Failed to parse MongoDB Url"); - // Set the server_api field of the client_options object to set the version of the Stable API on the client + // Set the server_api field of the client_options object to set the version of the Stable API on the + // client let server_api = ServerApi::builder().version(ServerApiVersion::V1).build(); client_options.server_api = Some(server_api); // Get a handle to the cluster @@ -39,9 +38,9 @@ impl MongoDb { self.client.database("orchestrator").collection("jobs") } - /// Updates the job in the database optimistically. This means that the job is updated only if the - /// version of the job in the database is the same as the version of the job passed in. If the version - /// is different, the update fails. + /// Updates the job in the database optimistically. This means that the job is updated only if + /// the version of the job in the database is the same as the version of the job passed in. + /// If the version is different, the update fails. async fn update_job_optimistically(&self, current_job: &JobItem, update: Document) -> Result<()> { let filter = doc! { "id": current_job.id, diff --git a/crates/orchestrator/src/jobs/constants.rs b/crates/orchestrator/src/jobs/constants.rs index 47a2cae1..0da8587d 100644 --- a/crates/orchestrator/src/jobs/constants.rs +++ b/crates/orchestrator/src/jobs/constants.rs @@ -1,2 +1,3 @@ pub const JOB_PROCESS_ATTEMPT_METADATA_KEY: &str = "process_attempt_no"; pub const JOB_VERIFICATION_ATTEMPT_METADATA_KEY: &str = "verification_attempt_no"; +pub const JOB_METADATA_CAIRO_PIE_PATH_KEY: &str = "cairo_pie_path"; diff --git a/crates/orchestrator/src/jobs/da_job/mod.rs b/crates/orchestrator/src/jobs/da_job/mod.rs index 157a224e..0566beba 100644 --- a/crates/orchestrator/src/jobs/da_job/mod.rs +++ b/crates/orchestrator/src/jobs/da_job/mod.rs @@ -1,24 +1,24 @@ -use crate::config::Config; -use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; -use crate::jobs::Job; +use std::collections::HashMap; +use std::ops::{Add, Mul, Rem}; +use std::result::Result::{Err, Ok as OtherOk}; +use std::str::FromStr; + use async_trait::async_trait; use color_eyre::eyre::{eyre, Ok}; use color_eyre::Result; use lazy_static::lazy_static; use num_bigint::{BigUint, ToBigUint}; -use num_traits::Num; -use num_traits::Zero; -use std::ops::{Add, Mul, Rem}; -use std::result::Result::{Err, Ok as OtherOk}; -use std::str::FromStr; - +use num_traits::{Num, Zero}; // use starknet::core::types::{BlockId, FieldElement, MaybePendingStateUpdate, StateUpdate, StorageEntry}; use starknet::providers::Provider; -use std::collections::HashMap; use tracing::log; use uuid::Uuid; +use super::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use super::Job; +use crate::config::Config; + lazy_static! { /// EIP-4844 BLS12-381 modulus. /// @@ -229,7 +229,8 @@ async fn state_update_to_blob_data( let mut nonce = nonces.remove(&addr); - // @note: if nonce is null and there is some len of writes, make an api call to get the contract nonce for the block + // @note: if nonce is null and there is some len of writes, make an api call to get the contract + // nonce for the block if nonce.is_none() && !writes.is_empty() && addr != FieldElement::ONE { let get_current_nonce_result = config.starknet_client().get_nonce(BlockId::Number(block_no), addr).await; @@ -314,19 +315,21 @@ fn da_word(class_flag: bool, nonce_change: Option, num_changes: u6 #[cfg(test)] mod tests { - use super::*; - use crate::tests::common::init_config; + use std::fs; + use std::fs::File; + use std::io::Read; + use ::serde::{Deserialize, Serialize}; use httpmock::prelude::*; use majin_blob_core::blob; - use majin_blob_types::{serde, state_diffs::UnorderedEq}; + use majin_blob_types::serde; + use majin_blob_types::state_diffs::UnorderedEq; // use majin_blob_types::serde; use rstest::rstest; - use serde_json::json; - use std::fs; - use std::fs::File; - use std::io::Read; + + use super::*; + use crate::tests::common::init_config; #[rstest] #[case(false, 1, 1, "18446744073709551617")] @@ -373,7 +376,7 @@ mod tests { ) { let server = MockServer::start(); - let config = init_config(Some(format!("http://localhost:{}", server.port())), None, None, None).await; + let config = init_config(Some(format!("http://localhost:{}", server.port())), None, None, None, None).await; get_nonce_attached(&server, nonce_file_path); diff --git a/crates/orchestrator/src/jobs/mod.rs b/crates/orchestrator/src/jobs/mod.rs index 8e357a0a..c2576480 100644 --- a/crates/orchestrator/src/jobs/mod.rs +++ b/crates/orchestrator/src/jobs/mod.rs @@ -1,20 +1,23 @@ -use crate::config::{config, Config}; -use crate::jobs::constants::{JOB_PROCESS_ATTEMPT_METADATA_KEY, JOB_VERIFICATION_ATTEMPT_METADATA_KEY}; -use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; -use crate::queue::job_queue::{add_job_to_process_queue, add_job_to_verification_queue}; +use std::collections::HashMap; +use std::time::Duration; + use async_trait::async_trait; use color_eyre::eyre::eyre; use color_eyre::Result; -use std::collections::HashMap; -use std::time::Duration; use tracing::log; use uuid::Uuid; +use crate::config::{config, Config}; +use crate::jobs::constants::{JOB_PROCESS_ATTEMPT_METADATA_KEY, JOB_VERIFICATION_ATTEMPT_METADATA_KEY}; +use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use crate::queue::job_queue::{add_job_to_process_queue, add_job_to_verification_queue}; + pub mod constants; pub mod da_job; -mod register_proof_job; +pub mod prover_job; +pub mod register_proof_job; pub mod snos_job; -mod state_update_job; +pub mod state_update_job; pub mod types; /// The Job trait is used to define the methods that a job @@ -68,8 +71,8 @@ pub async fn create_job(job_type: JobType, internal_id: String, metadata: HashMa Ok(()) } -/// Processes the job, increments the process attempt count and updates the status of the job in the DB. -/// It then adds the job to the verification queue. +/// Processes the job, increments the process attempt count and updates the status of the job in the +/// DB. It then adds the job to the verification queue. pub async fn process_job(id: Uuid) -> Result<()> { let config = config().await; let job = get_job(id).await?; @@ -86,7 +89,8 @@ pub async fn process_job(id: Uuid) -> Result<()> { } } // this updates the version of the job. this ensures that if another thread was about to process - // the same job, it would fail to update the job in the database because the version would be outdated + // the same job, it would fail to update the job in the database because the version would be + // outdated config.database().update_job_status(&job, JobStatus::LockedForProcessing).await?; let job_handler = get_job_handler(&job.job_type); @@ -104,9 +108,10 @@ pub async fn process_job(id: Uuid) -> Result<()> { Ok(()) } -/// Verifies the job and updates the status of the job in the DB. If the verification fails, it retries -/// processing the job if the max attempts have not been exceeded. If the max attempts have been exceeded, -/// it marks the job as timedout. If the verification is still pending, it pushes the job back to the queue. +/// Verifies the job and updates the status of the job in the DB. If the verification fails, it +/// retries processing the job if the max attempts have not been exceeded. If the max attempts have +/// been exceeded, it marks the job as timedout. If the verification is still pending, it pushes the +/// job back to the queue. pub async fn verify_job(id: Uuid) -> Result<()> { let config = config().await; let job = get_job(id).await?; diff --git a/crates/orchestrator/src/jobs/prover_job/mod.rs b/crates/orchestrator/src/jobs/prover_job/mod.rs new file mode 100644 index 00000000..d4df305a --- /dev/null +++ b/crates/orchestrator/src/jobs/prover_job/mod.rs @@ -0,0 +1,78 @@ +use std::collections::HashMap; +use std::path::PathBuf; +use std::str::FromStr; + +use async_trait::async_trait; +use cairo_vm::vm::runners::cairo_pie::CairoPie; +use color_eyre::eyre::eyre; +use color_eyre::Result; +use prover_client_interface::{Task, TaskStatus}; +use tracing::log::log; +use tracing::log::Level::Error; +use uuid::Uuid; + +use super::constants::JOB_METADATA_CAIRO_PIE_PATH_KEY; +use super::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use super::Job; +use crate::config::Config; + +pub struct ProverJob; + +#[async_trait] +impl Job for ProverJob { + async fn create_job( + &self, + _config: &Config, + internal_id: String, + metadata: HashMap, + ) -> Result { + if !metadata.contains_key(JOB_METADATA_CAIRO_PIE_PATH_KEY) { + return Err(eyre!("Cairo PIE path is not specified (prover job #{})", internal_id)); + } + Ok(JobItem { + id: Uuid::new_v4(), + internal_id, + job_type: JobType::ProofCreation, + status: JobStatus::Created, + external_id: String::new().into(), + metadata, + version: 0, + }) + } + + async fn process_job(&self, config: &Config, job: &JobItem) -> Result { + // TODO: allow to donwload PIE from S3 + let cairo_pie_path: PathBuf = job + .metadata + .get(JOB_METADATA_CAIRO_PIE_PATH_KEY) + .map(|s| PathBuf::from_str(s)) + .ok_or_else(|| eyre!("Cairo PIE path is not specified (prover job #{})", job.internal_id))??; + let cairo_pie = CairoPie::read_zip_file(&cairo_pie_path).unwrap(); + let external_id = config.prover_client().submit_task(Task::CairoPie(cairo_pie)).await?; + Ok(external_id) + } + + async fn verify_job(&self, config: &Config, job: &JobItem) -> Result { + let task_id: String = job.external_id.unwrap_string()?.into(); + match config.prover_client().get_task_status(&task_id).await? { + TaskStatus::Processing => Ok(JobVerificationStatus::Pending), + TaskStatus::Succeeded => Ok(JobVerificationStatus::Verified), + TaskStatus::Failed(err) => { + log!(Error, "Prover job #{} failed: {}", job.internal_id, err); + Ok(JobVerificationStatus::Rejected) + } + } + } + + fn max_process_attempts(&self) -> u64 { + 1 + } + + fn max_verification_attempts(&self) -> u64 { + 1 + } + + fn verification_polling_delay_seconds(&self) -> u64 { + 60 + } +} diff --git a/crates/orchestrator/src/jobs/register_proof_job/mod.rs b/crates/orchestrator/src/jobs/register_proof_job/mod.rs index 5f7a36d4..2e9482f7 100644 --- a/crates/orchestrator/src/jobs/register_proof_job/mod.rs +++ b/crates/orchestrator/src/jobs/register_proof_job/mod.rs @@ -1,11 +1,13 @@ -use crate::config::Config; -use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; -use crate::jobs::Job; +use std::collections::HashMap; + use async_trait::async_trait; use color_eyre::Result; -use std::collections::HashMap; use uuid::Uuid; +use crate::config::Config; +use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use crate::jobs::Job; + pub struct RegisterProofJob; #[async_trait] diff --git a/crates/orchestrator/src/jobs/snos_job/mod.rs b/crates/orchestrator/src/jobs/snos_job/mod.rs index 50fe0013..13600d4b 100644 --- a/crates/orchestrator/src/jobs/snos_job/mod.rs +++ b/crates/orchestrator/src/jobs/snos_job/mod.rs @@ -1,11 +1,13 @@ -use crate::config::Config; -use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; -use crate::jobs::Job; +use std::collections::HashMap; + use async_trait::async_trait; use color_eyre::Result; -use std::collections::HashMap; use uuid::Uuid; +use crate::config::Config; +use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use crate::jobs::Job; + pub struct SnosJob; #[async_trait] diff --git a/crates/orchestrator/src/jobs/state_update_job/mod.rs b/crates/orchestrator/src/jobs/state_update_job/mod.rs index 384eaf71..6eb14bc8 100644 --- a/crates/orchestrator/src/jobs/state_update_job/mod.rs +++ b/crates/orchestrator/src/jobs/state_update_job/mod.rs @@ -1,11 +1,13 @@ -use crate::config::Config; -use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; -use crate::jobs::Job; +use std::collections::HashMap; + use async_trait::async_trait; use color_eyre::Result; -use std::collections::HashMap; use uuid::Uuid; +use crate::config::Config; +use crate::jobs::types::{JobItem, JobStatus, JobType, JobVerificationStatus}; +use crate::jobs::Job; + pub struct StateUpdateJob; #[async_trait] diff --git a/crates/orchestrator/src/jobs/types.rs b/crates/orchestrator/src/jobs/types.rs index 7640ccad..15bb4679 100644 --- a/crates/orchestrator/src/jobs/types.rs +++ b/crates/orchestrator/src/jobs/types.rs @@ -1,9 +1,11 @@ -use color_eyre::{eyre::eyre, Result}; +use std::collections::HashMap; + +use color_eyre::eyre::eyre; +use color_eyre::Result; use da_client_interface::DaVerificationStatus; // TODO: job types shouldn't depend on mongodb use mongodb::bson::serde_helpers::uuid_1_as_binary; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use uuid::Uuid; /// An external id. @@ -77,7 +79,7 @@ pub enum JobType { /// Verifying the proof on the base layer ProofRegistration, /// Updaing the state root on the base layer - StateUpdation, + StateTransition, } #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, PartialOrd)] diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index 618c8481..4a19222d 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -12,8 +12,6 @@ pub mod jobs; pub mod queue; /// Contains the routes for the service pub mod routes; -/// Contains the utils -pub mod utils; /// Contains workers which act like cron jobs pub mod workers; diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 4df82c94..27bdf573 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -2,12 +2,12 @@ use dotenvy::dotenv; use orchestrator::config::config; use orchestrator::queue::init_consumers; use orchestrator::routes::app_router; -use orchestrator::utils::env_utils::get_env_var_or_default; use orchestrator::workers::proof_registration::ProofRegistrationWorker; use orchestrator::workers::proving::ProvingWorker; use orchestrator::workers::snos::SnosWorker; use orchestrator::workers::update_state::UpdateStateWorker; use orchestrator::workers::*; +use utils::env_utils::get_env_var_or_default; /// Start the server #[tokio::main] diff --git a/crates/orchestrator/src/queue/job_queue.rs b/crates/orchestrator/src/queue/job_queue.rs index 709d3aa8..9432276f 100644 --- a/crates/orchestrator/src/queue/job_queue.rs +++ b/crates/orchestrator/src/queue/job_queue.rs @@ -1,15 +1,17 @@ -use crate::config::config; -use crate::jobs::{process_job, verify_job}; +use std::future::Future; +use std::time::Duration; + use color_eyre::eyre::eyre; use color_eyre::Result; use omniqueue::QueueError; use serde::{Deserialize, Serialize}; -use std::future::Future; -use std::time::Duration; use tokio::time::sleep; use tracing::log; use uuid::Uuid; +use crate::config::config; +use crate::jobs::{process_job, verify_job}; + const JOB_PROCESSING_QUEUE: &str = "madara_orchestrator_job_processing_queue"; const JOB_VERIFICATION_QUEUE: &str = "madara_orchestrator_job_verification_queue"; diff --git a/crates/orchestrator/src/queue/mod.rs b/crates/orchestrator/src/queue/mod.rs index 74a9808a..01829892 100644 --- a/crates/orchestrator/src/queue/mod.rs +++ b/crates/orchestrator/src/queue/mod.rs @@ -1,12 +1,12 @@ pub mod job_queue; pub mod sqs; +use std::time::Duration; + use async_trait::async_trait; use color_eyre::Result; -use omniqueue::{Delivery, QueueError}; - use mockall::automock; -use std::time::Duration; +use omniqueue::{Delivery, QueueError}; /// The QueueProvider trait is used to define the methods that a queue /// should implement to be used as a queue for the orchestrator. The diff --git a/crates/orchestrator/src/queue/sqs/mod.rs b/crates/orchestrator/src/queue/sqs/mod.rs index 3f9d183a..0ba901fd 100644 --- a/crates/orchestrator/src/queue/sqs/mod.rs +++ b/crates/orchestrator/src/queue/sqs/mod.rs @@ -1,9 +1,11 @@ -use crate::queue::QueueProvider; +use std::time::Duration; + use async_trait::async_trait; use color_eyre::Result; use omniqueue::backends::{SqsBackend, SqsConfig, SqsConsumer, SqsProducer}; use omniqueue::{Delivery, QueueError}; -use std::time::Duration; + +use crate::queue::QueueProvider; pub struct SqsQueue; #[async_trait] diff --git a/crates/orchestrator/src/routes.rs b/crates/orchestrator/src/routes.rs index 41e1803a..39d8f3d4 100644 --- a/crates/orchestrator/src/routes.rs +++ b/crates/orchestrator/src/routes.rs @@ -1,9 +1,10 @@ -use crate::controllers::jobs_controller; use axum::http::StatusCode; use axum::response::IntoResponse; use axum::routing::{get, post}; use axum::Router; +use crate::controllers::jobs_controller; + pub fn app_router() -> Router { Router::new() .route("/health", get(root)) diff --git a/crates/orchestrator/src/tests/artifacts/fibonacci.zip b/crates/orchestrator/src/tests/artifacts/fibonacci.zip new file mode 100644 index 0000000000000000000000000000000000000000..b5943536870ae352374926593e2580178826de0d GIT binary patch literal 1514 zcmWIWW@Zs#U|`^2uyWrNvG>CD@1j7SFc9+qaan3nab|v=URH5_-s&(fzcV_g`t?uv zgl-Blx?&u%ij^S%Mf<`V3to9KGBB7hGcfQ1wdbamB&H;mB!cy?opjpoumg|l|I}-n zKir=8kIPBJ!C_)X)Yg?P4u8w%T`{xGL08rSF+QgPP)+2+`;*38rP{7YmklO^pyyywh`0lJ&@>o@;k3G!ISS zn0M0NW#PJ;kFF+Zm~GB_d8uJ`=G2?oN&(*zq$V~s#LuZKcJj3hx#StjLF=Y(x%l{rD+)Y~`DJ8dglW#hl-OGP_U(UbtF%Nd?UH?+>CVu;0 zi?iXTJ?H=aUizf)$BC{dC%?Y`dpT38>@im-cjlBQ3+KzlebbiEmxx*U_cE92_A8|- z++}lizRsu<*`o2G#?VT3m-NBAe}8hL272cmEx{+iKo{U+VBi7=dTxGErCw5IUhlP& zLAL`0S|4gZ<`g!THkir5siOGV`gA8IxI9Q~Tm74zX6rnDmK20P-BLC8UB$ZF zSLTM5?#{a#w;|8I;$F>%J&qstu-;L4`gw(=etm8Gj-6A^EBU>x#`r;nOq<{XP4aF1+O1o z`M#-t@jmg{$90$GysB#aCsvT8|F!t=iuor6+-vnq|4)@(o|B__6XD8I>7w$eJ`{2s= zx3WMZ-`M({{it~3zWbep1@jAD{roNobkR4~<+BiLlV^fAtqRlf@bTAWGs-HRBINDWWi;8#Tj9^G zD74(YtY~JC575v+AeI2))QZ&PQjqEKMXAO4rA5i9#o&ytd&-dOfC7)h#oZ}o8M{UH z_++RCI0_q{3^=m;`ohQm_b~QkT@iUy*&;f1)faJj%`NxD9(;?@F-{BHZ6T;uX!c5g zy(7B#@O+0m&;I~jz{sS>jJtpWIs*)rG=eDP;tE|Sdf5cg$H1_p(H_V|DXGx4qUSJ# z)-Aw%mzxU9Z|M5aV*{b@0I-0>7B?6spanU?1X&i`5fI?b$_7%&3WTOW7tRN&WncgR DephfO literal 0 HcmV?d00001 diff --git a/crates/orchestrator/src/tests/common/mod.rs b/crates/orchestrator/src/tests/common/mod.rs index a074eb48..f2ce742e 100644 --- a/crates/orchestrator/src/tests/common/mod.rs +++ b/crates/orchestrator/src/tests/common/mod.rs @@ -1,30 +1,30 @@ pub mod constants; -use constants::*; -use rstest::*; - use std::collections::HashMap; use std::sync::Arc; -use crate::{ - config::Config, - jobs::types::{ExternalId, JobItem, JobStatus::Created, JobType::DataSubmission}, -}; - -use crate::database::MockDatabase; -use crate::queue::MockQueueProvider; use ::uuid::Uuid; +use constants::*; use da_client_interface::MockDaClient; +use prover_client_interface::MockProverClient; +use rstest::*; use starknet::providers::jsonrpc::HttpTransport; use starknet::providers::JsonRpcClient; - use url::Url; +use crate::config::Config; +use crate::database::MockDatabase; +use crate::jobs::types::JobStatus::Created; +use crate::jobs::types::JobType::DataSubmission; +use crate::jobs::types::{ExternalId, JobItem}; +use crate::queue::MockQueueProvider; + pub async fn init_config( rpc_url: Option, database: Option, queue: Option, da_client: Option, + prover_client: Option, ) -> Config { let _ = tracing_subscriber::fmt().with_max_level(tracing::Level::INFO).with_target(false).try_init(); @@ -32,11 +32,12 @@ pub async fn init_config( let database = database.unwrap_or_default(); let queue = queue.unwrap_or_default(); let da_client = da_client.unwrap_or_default(); + let prover_client = prover_client.unwrap_or_default(); // init starknet client let provider = JsonRpcClient::new(HttpTransport::new(Url::parse(rpc_url.as_str()).expect("Failed to parse URL"))); - Config::new(Arc::new(provider), Box::new(da_client), Box::new(database), Box::new(queue)) + Config::new(Arc::new(provider), Box::new(da_client), Box::new(prover_client), Box::new(database), Box::new(queue)) } #[fixture] diff --git a/crates/orchestrator/src/tests/jobs/da_job/mod.rs b/crates/orchestrator/src/tests/jobs/da_job/mod.rs index d6548f2d..6aab1ea3 100644 --- a/crates/orchestrator/src/tests/jobs/da_job/mod.rs +++ b/crates/orchestrator/src/tests/jobs/da_job/mod.rs @@ -1,30 +1,22 @@ -use rstest::*; -use starknet::core::types::StateUpdate; - use std::collections::HashMap; +use da_client_interface::{DaVerificationStatus, MockDaClient}; use httpmock::prelude::*; +use rstest::*; use serde_json::json; - -use super::super::common::{ - constants::{ETHEREUM_MAX_BLOB_PER_TXN, ETHEREUM_MAX_BYTES_PER_BLOB}, - default_job_item, init_config, -}; -use starknet_core::types::{FieldElement, MaybePendingStateUpdate, StateDiff}; +use starknet_core::types::{FieldElement, MaybePendingStateUpdate, StateDiff, StateUpdate}; use uuid::Uuid; -use crate::jobs::types::ExternalId; -use crate::jobs::{ - da_job::DaJob, - types::{JobItem, JobStatus, JobType}, - Job, -}; -use da_client_interface::{DaVerificationStatus, MockDaClient}; +use super::super::common::constants::{ETHEREUM_MAX_BLOB_PER_TXN, ETHEREUM_MAX_BYTES_PER_BLOB}; +use super::super::common::{default_job_item, init_config}; +use crate::jobs::da_job::DaJob; +use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; +use crate::jobs::Job; #[rstest] #[tokio::test] async fn test_create_job() { - let config = init_config(None, None, None, None).await; + let config = init_config(None, None, None, None, None).await; let job = DaJob.create_job(&config, String::from("0"), HashMap::new()).await; assert!(job.is_ok()); @@ -44,7 +36,7 @@ async fn test_verify_job(#[from(default_job_item)] job_item: JobItem) { let mut da_client = MockDaClient::new(); da_client.expect_verify_inclusion().times(1).returning(|_| Ok(DaVerificationStatus::Verified)); - let config = init_config(None, None, None, Some(da_client)).await; + let config = init_config(None, None, None, Some(da_client), None).await; assert!(DaJob.verify_job(&config, &job_item).await.is_ok()); } @@ -60,7 +52,8 @@ async fn test_process_job() { da_client.expect_max_bytes_per_blob().times(1).returning(move || ETHEREUM_MAX_BYTES_PER_BLOB); da_client.expect_max_blob_per_txn().times(1).returning(move || ETHEREUM_MAX_BLOB_PER_TXN); - let config = init_config(Some(format!("http://localhost:{}", server.port())), None, None, Some(da_client)).await; + let config = + init_config(Some(format!("http://localhost:{}", server.port())), None, None, Some(da_client), None).await; let state_update = MaybePendingStateUpdate::Update(StateUpdate { block_hash: FieldElement::default(), new_root: FieldElement::default(), diff --git a/crates/orchestrator/src/tests/jobs/mod.rs b/crates/orchestrator/src/tests/jobs/mod.rs index 487732dd..f99e374a 100644 --- a/crates/orchestrator/src/tests/jobs/mod.rs +++ b/crates/orchestrator/src/tests/jobs/mod.rs @@ -3,6 +3,9 @@ use rstest::rstest; #[cfg(test)] pub mod da_job; +#[cfg(test)] +pub mod prover_job; + #[rstest] #[tokio::test] async fn create_job_fails_job_already_exists() { diff --git a/crates/orchestrator/src/tests/jobs/prover_job/mod.rs b/crates/orchestrator/src/tests/jobs/prover_job/mod.rs new file mode 100644 index 00000000..17e28dc4 --- /dev/null +++ b/crates/orchestrator/src/tests/jobs/prover_job/mod.rs @@ -0,0 +1,78 @@ +use std::collections::HashMap; + +use httpmock::prelude::*; +use prover_client_interface::{MockProverClient, TaskStatus}; +use rstest::*; +use uuid::Uuid; + +use super::super::common::{default_job_item, init_config}; +use crate::jobs::constants::JOB_METADATA_CAIRO_PIE_PATH_KEY; +use crate::jobs::prover_job::ProverJob; +use crate::jobs::types::{JobItem, JobStatus, JobType}; +use crate::jobs::Job; + +#[rstest] +#[tokio::test] +async fn test_create_job() { + let config = init_config(None, None, None, None, None).await; + let job = ProverJob + .create_job( + &config, + String::from("0"), + HashMap::from([(JOB_METADATA_CAIRO_PIE_PATH_KEY.into(), "pie.zip".into())]), + ) + .await; + assert!(job.is_ok()); + + let job = job.unwrap(); + + let job_type = job.job_type; + assert_eq!(job_type, JobType::ProofCreation, "job_type should be ProofCreation"); + assert!(!(job.id.is_nil()), "id should not be nil"); + assert_eq!(job.status, JobStatus::Created, "status should be Created"); + assert_eq!(job.version, 0_i32, "version should be 0"); + assert_eq!(job.external_id.unwrap_string().unwrap(), String::new(), "external_id should be empty string"); +} + +#[rstest] +#[tokio::test] +async fn test_verify_job(#[from(default_job_item)] job_item: JobItem) { + let mut prover_client = MockProverClient::new(); + prover_client.expect_get_task_status().times(1).returning(|_| Ok(TaskStatus::Succeeded)); + + let config = init_config(None, None, None, None, Some(prover_client)).await; + assert!(ProverJob.verify_job(&config, &job_item).await.is_ok()); +} + +#[rstest] +#[tokio::test] +async fn test_process_job() { + let server = MockServer::start(); + + let mut prover_client = MockProverClient::new(); + prover_client.expect_submit_task().times(1).returning(|_| Ok("task_id".to_string())); + + let config = + init_config(Some(format!("http://localhost:{}", server.port())), None, None, None, Some(prover_client)).await; + + let cairo_pie_path = format!("{}/src/tests/artifacts/fibonacci.zip", env!("CARGO_MANIFEST_DIR")); + + assert_eq!( + ProverJob + .process_job( + &config, + &JobItem { + id: Uuid::default(), + internal_id: "0".into(), + job_type: JobType::ProofCreation, + status: JobStatus::Created, + external_id: String::new().into(), + metadata: HashMap::from([(JOB_METADATA_CAIRO_PIE_PATH_KEY.into(), cairo_pie_path)]), + version: 0, + } + ) + .await + .unwrap(), + "task_id".to_string() + ); +} diff --git a/crates/orchestrator/src/tests/server/mod.rs b/crates/orchestrator/src/tests/server/mod.rs index 79b427e6..bab1a5d0 100644 --- a/crates/orchestrator/src/tests/server/mod.rs +++ b/crates/orchestrator/src/tests/server/mod.rs @@ -1,18 +1,19 @@ -use std::{io::Read, net::SocketAddr}; +use std::io::Read; +use std::net::SocketAddr; use axum::http::StatusCode; - -use hyper::{body::Buf, Body, Request}; - +use hyper::body::Buf; +use hyper::{Body, Request}; use rstest::*; - -use crate::{queue::init_consumers, routes::app_router, utils::env_utils::get_env_var_or_default}; +use utils::env_utils::get_env_var_or_default; use super::common::init_config; +use crate::queue::init_consumers; +use crate::routes::app_router; #[fixture] pub async fn setup_server() -> SocketAddr { - let _config = init_config(Some("http://localhost:9944".to_string()), None, None, None).await; + let _config = init_config(Some("http://localhost:9944".to_string()), None, None, None, None).await; let host = get_env_var_or_default("HOST", "127.0.0.1"); let port = get_env_var_or_default("PORT", "3000").parse::().expect("PORT must be a u16"); diff --git a/crates/orchestrator/src/tests/workers/mod.rs b/crates/orchestrator/src/tests/workers/mod.rs index 18f0ee87..cd3a1243 100644 --- a/crates/orchestrator/src/tests/workers/mod.rs +++ b/crates/orchestrator/src/tests/workers/mod.rs @@ -1,3 +1,13 @@ +use std::collections::HashMap; +use std::error::Error; + +use da_client_interface::MockDaClient; +use httpmock::MockServer; +use mockall::predicate::eq; +use rstest::rstest; +use serde_json::json; +use uuid::Uuid; + use crate::config::config_force_init; use crate::database::MockDatabase; use crate::jobs::types::{ExternalId, JobItem, JobStatus, JobType}; @@ -5,14 +15,6 @@ use crate::queue::MockQueueProvider; use crate::tests::common::init_config; use crate::workers::snos::SnosWorker; use crate::workers::Worker; -use da_client_interface::MockDaClient; -use httpmock::MockServer; -use mockall::predicate::eq; -use rstest::rstest; -use serde_json::json; -use std::collections::HashMap; -use std::error::Error; -use uuid::Uuid; #[rstest] #[case(false)] @@ -69,7 +71,8 @@ async fn test_snos_worker(#[case] db_val: bool) -> Result<(), Box> { let rpc_response_block_number = block; let response = json!({ "id": 1,"jsonrpc":"2.0","result": rpc_response_block_number }); let config = - init_config(Some(format!("http://localhost:{}", server.port())), Some(db), Some(queue), Some(da_client)).await; + init_config(Some(format!("http://localhost:{}", server.port())), Some(db), Some(queue), Some(da_client), None) + .await; config_force_init(config).await; // mocking block call diff --git a/crates/orchestrator/src/utils/env_utils.rs b/crates/orchestrator/src/utils/env_utils.rs deleted file mode 100644 index 78e11609..00000000 --- a/crates/orchestrator/src/utils/env_utils.rs +++ /dev/null @@ -1,13 +0,0 @@ -use color_eyre::Result; - -pub fn get_env_var(key: &str) -> Result { - std::env::var(key).map_err(|e| e.into()) -} - -pub fn get_env_var_or_panic(key: &str) -> String { - get_env_var(key).unwrap_or_else(|e| panic!("Failed to get env var {}: {}", key, e)) -} - -pub fn get_env_var_or_default(key: &str, default: &str) -> String { - get_env_var(key).unwrap_or(default.to_string()) -} diff --git a/crates/orchestrator/src/utils/mod.rs b/crates/orchestrator/src/utils/mod.rs deleted file mode 100644 index 6a65fdce..00000000 --- a/crates/orchestrator/src/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod env_utils; diff --git a/crates/orchestrator/src/workers/mod.rs b/crates/orchestrator/src/workers/mod.rs index 7d198c0c..ba833b7e 100644 --- a/crates/orchestrator/src/workers/mod.rs +++ b/crates/orchestrator/src/workers/mod.rs @@ -1,6 +1,7 @@ -use async_trait::async_trait; use std::error::Error; +use async_trait::async_trait; + pub mod proof_registration; pub mod proving; pub mod snos; diff --git a/crates/orchestrator/src/workers/proof_registration.rs b/crates/orchestrator/src/workers/proof_registration.rs index a51bd412..ea6e5544 100644 --- a/crates/orchestrator/src/workers/proof_registration.rs +++ b/crates/orchestrator/src/workers/proof_registration.rs @@ -1,7 +1,9 @@ -use crate::workers::Worker; -use async_trait::async_trait; use std::error::Error; +use async_trait::async_trait; + +use crate::workers::Worker; + pub struct ProofRegistrationWorker; #[async_trait] diff --git a/crates/orchestrator/src/workers/proving.rs b/crates/orchestrator/src/workers/proving.rs index 61bc19c2..608d211a 100644 --- a/crates/orchestrator/src/workers/proving.rs +++ b/crates/orchestrator/src/workers/proving.rs @@ -1,7 +1,9 @@ -use crate::workers::Worker; -use async_trait::async_trait; use std::error::Error; +use async_trait::async_trait; + +use crate::workers::Worker; + pub struct ProvingWorker; #[async_trait] diff --git a/crates/orchestrator/src/workers/snos.rs b/crates/orchestrator/src/workers/snos.rs index f116cf67..139d79e1 100644 --- a/crates/orchestrator/src/workers/snos.rs +++ b/crates/orchestrator/src/workers/snos.rs @@ -1,11 +1,13 @@ +use std::collections::HashMap; +use std::error::Error; + +use async_trait::async_trait; +use starknet::providers::Provider; + use crate::config::config; use crate::jobs::create_job; use crate::jobs::types::JobType; use crate::workers::Worker; -use async_trait::async_trait; -use starknet::providers::Provider; -use std::collections::HashMap; -use std::error::Error; pub struct SnosWorker; diff --git a/crates/orchestrator/src/workers/update_state.rs b/crates/orchestrator/src/workers/update_state.rs index faca16ab..7933a675 100644 --- a/crates/orchestrator/src/workers/update_state.rs +++ b/crates/orchestrator/src/workers/update_state.rs @@ -1,7 +1,9 @@ -use crate::workers::Worker; -use async_trait::async_trait; use std::error::Error; +use async_trait::async_trait; + +use crate::workers::Worker; + pub struct UpdateStateWorker; #[async_trait] diff --git a/crates/prover-services/gps-fact-checker/Cargo.toml b/crates/prover-services/gps-fact-checker/Cargo.toml new file mode 100644 index 00000000..fb0e7478 --- /dev/null +++ b/crates/prover-services/gps-fact-checker/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "gps-fact-checker" +version.workspace = true +edition.workspace = true + +[dependencies] +alloy = { workspace = true, features = [ + "sol-types", + "json", + "contract", + "providers", + "rpc-client", + "transport-http", + "reqwest", +] } +async-trait.workspace = true +cairo-vm.workspace = true +itertools.workspace = true +starknet.workspace = true +thiserror.workspace = true +url.workspace = true +utils.workspace = true diff --git a/crates/prover-services/gps-fact-checker/src/error.rs b/crates/prover-services/gps-fact-checker/src/error.rs new file mode 100644 index 00000000..166011be --- /dev/null +++ b/crates/prover-services/gps-fact-checker/src/error.rs @@ -0,0 +1,53 @@ +use cairo_vm::program_hash::ProgramHashError; + +#[derive(Debug, thiserror::Error)] +pub enum FactCheckerError { + #[error("Fact registry call failed: {0}")] + FactRegistry(#[source] alloy::contract::Error), + #[error("Failed to compute program hash: {0}")] + ProgramHashCompute(#[from] ProgramHashError), + #[error("There is no additional data for the output builtin in Cairo PIE")] + OutputBuiltinNoAdditionalData, + #[error("There is no segment info for the output builtin in Cairo PIE")] + OutputBuiltinNoSegmentInfo, + #[error("Tree structure length is not even")] + TreeStructureLenOdd, + #[error("Tree structure is empty")] + TreeStructureEmpty, + #[error("Tree structure is too large")] + TreeStructureTooLarge, + #[error("Tree structure contains invalid values")] + TreeStructureInvalid, + #[error("Output pages length is unexpected")] + OutputPagesLenUnexpected, + #[error("Output page {0} has invalid start {1} (expected 0 < x < {2})")] + OutputPagesInvalidStart(usize, usize, usize), + #[error("Output page {0} has expected start {1} (expected{2})")] + OutputPagesUnexpectedStart(usize, usize, usize), + #[error("Output page {0} has invalid size {1} (expected 0 < x < {2})")] + OutputPagesInvalidSize(usize, usize, usize), + #[error("Output page {0} has unexpected id (expected {1})")] + OutputPagesUnexpectedId(usize, usize), + #[error("Output pages cover only {0} out of {1} output elements")] + OutputPagesUncoveredOutput(usize, usize), + #[error("Output segment is not found in the memory")] + OutputSegmentNotFound, + #[error("Output segment does not fit into the memory")] + OutputSegmentInvalidRange, + #[error("Output segment contains inconsistent offset {0} (expected {1})")] + OutputSegmentInconsistentOffset(usize, usize), + #[error("Output segment contains unexpected relocatable at position {0}")] + OutputSegmentUnexpectedRelocatable(usize), + #[error("Tree structure: pages count {0} is in invalid range (expected <= {1})")] + TreeStructurePagesCountOutOfRange(usize, usize), + #[error("Tree structure: nodes count {0} is in invalid range (expected <= {1})")] + TreeStructureNodesCountOutOfRange(usize, usize), + #[error("Tree structure: node stack contains more than one node")] + TreeStructureRootInvalid, + #[error("Tree structure: {0} pages were not processed")] + TreeStructurePagesNotProcessed(usize), + #[error("Tree structure: end offset {0} does not match the output length {1}")] + TreeStructureEndOffsetInvalid(usize, usize), + #[error("Tree structure: root offset {0} does not match the output length {1}")] + TreeStructureRootOffsetInvalid(usize, usize), +} diff --git a/crates/prover-services/gps-fact-checker/src/fact_info.rs b/crates/prover-services/gps-fact-checker/src/fact_info.rs new file mode 100644 index 00000000..e7156ef9 --- /dev/null +++ b/crates/prover-services/gps-fact-checker/src/fact_info.rs @@ -0,0 +1,98 @@ +//! Fact info structure and helpers. +//! +//! Port of https://github.com/starkware-libs/cairo-lang/blob/master/src/starkware/cairo/bootloaders/generate_fact.py + +use alloy::primitives::{keccak256, B256}; +use cairo_vm::program_hash::compute_program_hash_chain; +use cairo_vm::types::builtin_name::BuiltinName; +use cairo_vm::types::relocatable::MaybeRelocatable; +use cairo_vm::vm::runners::cairo_pie::CairoPie; +use cairo_vm::Felt252; +use starknet::core::types::FieldElement; +use utils::ensure; + +use super::error::FactCheckerError; +use super::fact_node::generate_merkle_root; +use super::fact_topology::{get_fact_topology, FactTopology}; + +/// Default bootloader program version. +/// +/// https://github.com/starkware-libs/cairo-lang/blob/efa9648f57568aad8f8a13fbf027d2de7c63c2c0/src/starkware/cairo/bootloaders/hash_program.py#L11 +pub const BOOTLOADER_VERSION: usize = 0; + +pub struct FactInfo { + pub program_output: Vec, + pub fact_topology: FactTopology, + pub fact: B256, +} + +pub fn get_fact_info(cairo_pie: &CairoPie, program_hash: Option) -> Result { + let program_output = get_program_output(cairo_pie)?; + let fact_topology = get_fact_topology(cairo_pie, program_output.len())?; + let program_hash = match program_hash { + Some(hash) => hash, + None => compute_program_hash_chain(&cairo_pie.metadata.program, BOOTLOADER_VERSION)?, + }; + let output_root = generate_merkle_root(&program_output, &fact_topology)?; + let fact = keccak256([program_hash.to_bytes_be(), *output_root.node_hash].concat()); + Ok(FactInfo { program_output, fact_topology, fact }) +} + +pub fn get_program_output(cairo_pie: &CairoPie) -> Result, FactCheckerError> { + let segment_info = cairo_pie + .metadata + .builtin_segments + .get(&BuiltinName::output) + .ok_or(FactCheckerError::OutputBuiltinNoSegmentInfo)?; + + let segment_start = cairo_pie + .memory + .0 + .iter() + .enumerate() + .find_map(|(ptr, ((index, _), _))| if *index == segment_info.index as usize { Some(ptr) } else { None }) + .ok_or(FactCheckerError::OutputSegmentNotFound)?; + + let mut output = Vec::with_capacity(segment_info.size); + let mut expected_offset = 0; + + #[allow(clippy::explicit_counter_loop)] + for i in segment_start..segment_start + segment_info.size { + let ((_, offset), value) = cairo_pie.memory.0.get(i).ok_or(FactCheckerError::OutputSegmentInvalidRange)?; + + ensure!( + *offset == expected_offset, + FactCheckerError::OutputSegmentInconsistentOffset(*offset, expected_offset) + ); + match value { + MaybeRelocatable::Int(felt) => output.push(*felt), + MaybeRelocatable::RelocatableValue(_) => { + return Err(FactCheckerError::OutputSegmentUnexpectedRelocatable(*offset)); + } + } + + expected_offset += 1; + } + + Ok(output) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use cairo_vm::vm::runners::cairo_pie::CairoPie; + + use super::get_fact_info; + + #[test] + fn test_fact_info() { + // Generated using the get_fact.py script + let expected_fact = "0xca15503f02f8406b599cb220879e842394f5cf2cef753f3ee430647b5981b782"; + let cairo_pie_path: PathBuf = + [env!("CARGO_MANIFEST_DIR"), "tests", "artifacts", "fibonacci.zip"].iter().collect(); + let cairo_pie = CairoPie::read_zip_file(&cairo_pie_path).unwrap(); + let fact_info = get_fact_info(&cairo_pie, None).unwrap(); + assert_eq!(expected_fact, fact_info.fact.to_string()); + } +} diff --git a/crates/prover-services/gps-fact-checker/src/fact_node.rs b/crates/prover-services/gps-fact-checker/src/fact_node.rs new file mode 100644 index 00000000..2d66fbac --- /dev/null +++ b/crates/prover-services/gps-fact-checker/src/fact_node.rs @@ -0,0 +1,117 @@ +//! Fact node structure and helpers. +//! +//! The fact of each task is stored as a (non-binary) Merkle tree. +//! Leaf nodes are labeled with the hash of their data. +//! Each non-leaf node is labeled as 1 + the hash of (node0, end0, node1, end1, ...) +//! where node* is a label of a child children and end* is the total number of data words up to +//! and including that node and its children (including the previous sibling nodes). +//! We add 1 to the result of the hash to prevent an attacker from using a preimage of a leaf node +//! as a preimage of a non-leaf hash and vice versa. +//! +//! The structure of the tree is passed as a list of pairs (n_pages, n_nodes), and the tree is +//! constructed using a stack of nodes (initialized to an empty stack) by repeating for each pair: +//! 1. Add #n_pages lead nodes to the stack. +//! 2. Pop the top #n_nodes, construct a parent node for them, and push it back to the stack. +//! After applying the steps above, the stack must contain exactly one node, which will +//! constitute the root of the Merkle tree. +//! +//! For example, [(2, 2)] will create a Merkle tree with a root and two direct children, while +//! [(3, 2), (0, 2)] will create a Merkle tree with a root whose left child is a leaf and +//! right child has two leaf children. +//! +//! Port of https://github.com/starkware-libs/cairo-lang/blob/master/src/starkware/cairo/bootloaders/compute_fact.py + +use alloy::primitives::{keccak256, B256}; +use cairo_vm::Felt252; +use itertools::Itertools; +use utils::ensure; + +use super::error::FactCheckerError; +use super::fact_topology::FactTopology; + +/// Node of the fact tree +#[derive(Debug, Clone)] +pub struct FactNode { + /// Page hash (leaf) or 1 + keccak{children} (non-leaf) + pub node_hash: B256, + /// Total number of data words up to that node (including it and its children) + pub end_offset: usize, + /// Page size + pub page_size: usize, + /// Child nodes + pub children: Vec, +} + +/// Generates the root of the output Merkle tree for the program fact computation. +/// +/// Basically it transforms the flat fact topology into a non-binary Merkle tree and then computes +/// its root, enriching the nodes with metadata such as page sizes and hashes. +pub fn generate_merkle_root( + program_output: &[Felt252], + fact_topology: &FactTopology, +) -> Result { + let FactTopology { tree_structure, mut page_sizes } = fact_topology.clone(); + + let mut end_offset: usize = 0; + let mut node_stack: Vec = Vec::with_capacity(page_sizes.len()); + let mut output_iter = program_output.iter(); + + for (n_pages, n_nodes) in tree_structure.into_iter().tuples() { + ensure!( + n_pages <= page_sizes.len(), + FactCheckerError::TreeStructurePagesCountOutOfRange(n_pages, page_sizes.len()) + ); + + // Push n_pages (leaves) to the stack + for _ in 0..n_pages { + let page_size = page_sizes.remove(0); + // Page size is already validated upon retrieving the topology + let page = output_iter.by_ref().take(page_size).map(|felt| felt.to_bytes_be().to_vec()).concat(); + let node_hash = keccak256(&page); + end_offset += page_size; + // Add lead node (no children) + node_stack.push(FactNode { node_hash, end_offset, page_size, children: vec![] }) + } + + ensure!( + n_nodes <= node_stack.len(), + FactCheckerError::TreeStructureNodesCountOutOfRange(n_nodes, node_stack.len()) + ); + + if n_nodes > 0 { + // Create a parent node to the last n_nodes in the head of the stack. + let children: Vec = node_stack.drain(node_stack.len() - n_nodes..).collect(); + let mut node_data = Vec::with_capacity(2 * 32 * children.len()); + let mut total_page_size = 0; + let mut child_end_offset = 0; + + for node in children.iter() { + node_data.copy_from_slice(node.node_hash.as_slice()); + node_data.copy_from_slice(&[0; 32 - (usize::BITS / 8) as usize]); // pad usize to 32 bytes + node_data.copy_from_slice(&node.page_size.to_be_bytes()); + total_page_size += node.page_size; + child_end_offset = node.end_offset; + } + + node_stack.push(FactNode { + node_hash: keccak256(&node_data), + end_offset: child_end_offset, + page_size: total_page_size, + children, + }) + } + + ensure!(node_stack.len() == 1, FactCheckerError::TreeStructureRootInvalid); + ensure!(page_sizes.is_empty(), FactCheckerError::TreeStructurePagesNotProcessed(page_sizes.len())); + ensure!( + end_offset == program_output.len(), + FactCheckerError::TreeStructureEndOffsetInvalid(end_offset, program_output.len()) + ); + ensure!( + node_stack[0].end_offset == program_output.len(), + FactCheckerError::TreeStructureRootOffsetInvalid(node_stack[0].end_offset, program_output.len(),) + ); + } + + Ok(node_stack.remove(0)) +} diff --git a/crates/prover-services/gps-fact-checker/src/fact_topology.rs b/crates/prover-services/gps-fact-checker/src/fact_topology.rs new file mode 100644 index 00000000..e2d7127c --- /dev/null +++ b/crates/prover-services/gps-fact-checker/src/fact_topology.rs @@ -0,0 +1,102 @@ +//! Fact topology type and helpers. +//! +//! Ported from https://github.com/starkware-libs/cairo-lang/blob/master/src/starkware/cairo/bootloaders/fact_topology.py + +use std::collections::HashMap; + +use cairo_vm::types::builtin_name::BuiltinName; +use cairo_vm::vm::runners::cairo_pie::{BuiltinAdditionalData, CairoPie, PublicMemoryPage}; +use utils::ensure; + +use super::error::FactCheckerError; + +pub const GPS_FACT_TOPOLOGY: &str = "gps_fact_topology"; + +/// Flattened fact tree +#[derive(Debug, Clone)] +pub struct FactTopology { + /// List of pairs (n_pages, n_nodes) + pub tree_structure: Vec, + /// Page sizes (pages are leaf nodes) + pub page_sizes: Vec, +} + +/// Returns the fact topology from the additional data of the output builtin. +pub fn get_fact_topology(cairo_pie: &CairoPie, output_size: usize) -> Result { + if let Some(BuiltinAdditionalData::Output(additional_data)) = cairo_pie.additional_data.0.get(&BuiltinName::output) + { + let tree_structure = match additional_data.attributes.get(GPS_FACT_TOPOLOGY) { + Some(tree_structure) => { + ensure!(!tree_structure.is_empty(), FactCheckerError::TreeStructureEmpty); + ensure!(tree_structure.len() % 2 == 0, FactCheckerError::TreeStructureLenOdd); + ensure!(tree_structure.len() <= 10, FactCheckerError::TreeStructureTooLarge); + ensure!(tree_structure.iter().all(|&x| x < 2 << 30), FactCheckerError::TreeStructureInvalid); + tree_structure.clone() + } + None => { + ensure!(additional_data.pages.is_empty(), FactCheckerError::OutputPagesLenUnexpected); + vec![1, 0] + } + }; + let page_sizes = get_page_sizes(&additional_data.pages, output_size)?; + Ok(FactTopology { tree_structure, page_sizes }) + } else { + Err(FactCheckerError::OutputBuiltinNoAdditionalData) + } +} + +/// Returns the sizes of the program output pages, given the pages dictionary that appears +/// in the additional attributes of the output builtin. +pub fn get_page_sizes( + pages: &HashMap, + output_size: usize, +) -> Result, FactCheckerError> { + let mut pages_list: Vec<(usize, usize, usize)> = + pages.iter().map(|(&id, page)| (id, page.start, page.size)).collect(); + pages_list.sort(); + + // The first page id is expected to be 1. + let mut expected_page_id = 1; + // We don't expect anything on its start value. + let mut expected_page_start = None; + + let mut page_sizes = Vec::with_capacity(pages_list.len() + 1); + // The size of page 0 is output_size if there are no other pages, or the start of page 1 otherwise. + page_sizes.push(output_size); + + for (page_id, page_start, page_size) in pages_list { + ensure!(page_id == expected_page_id, FactCheckerError::OutputPagesUnexpectedId(page_id, expected_page_id)); + + if page_id == 1 { + ensure!( + page_start > 0 && page_start < output_size, + FactCheckerError::OutputPagesInvalidStart(page_id, page_start, output_size) + ); + page_sizes[0] = page_start; + } else { + ensure!( + Some(page_start) == expected_page_start, + FactCheckerError::OutputPagesUnexpectedStart( + page_id, + page_start, + expected_page_start.unwrap_or_default(), + ) + ); + } + + ensure!( + page_size > 0 && page_size < output_size, + FactCheckerError::OutputPagesInvalidSize(page_id, page_size, output_size) + ); + expected_page_start = Some(page_start + page_size); + expected_page_id += 1; + + page_sizes.push(page_size); + } + + ensure!( + pages.is_empty() || expected_page_start == Some(output_size), + FactCheckerError::OutputPagesUncoveredOutput(expected_page_start.unwrap_or_default(), output_size) + ); + Ok(page_sizes) +} diff --git a/crates/prover-services/gps-fact-checker/src/lib.rs b/crates/prover-services/gps-fact-checker/src/lib.rs new file mode 100644 index 00000000..239e7556 --- /dev/null +++ b/crates/prover-services/gps-fact-checker/src/lib.rs @@ -0,0 +1,40 @@ +pub mod error; +pub mod fact_info; +pub mod fact_node; +pub mod fact_topology; + +use alloy::primitives::{Address, B256}; +use alloy::providers::{ProviderBuilder, RootProvider}; +use alloy::sol; +use alloy::transports::http::{Client, Http}; +use url::Url; + +use self::error::FactCheckerError; + +sol!( + #[allow(missing_docs)] + #[sol(rpc)] + FactRegistry, + "tests/artifacts/FactRegistry.json" +); + +pub struct FactChecker { + fact_registry: FactRegistry::FactRegistryInstance, +} + +type TransportT = Http; +type ProviderT = RootProvider; + +impl FactChecker { + pub fn new(rpc_node_url: Url, verifier_address: Address) -> Self { + let provider = ProviderBuilder::new().on_http(rpc_node_url); + let fact_registry = FactRegistry::new(verifier_address, provider); + Self { fact_registry } + } + + pub async fn is_valid(&self, fact: &B256) -> Result { + let FactRegistry::isValidReturn { _0 } = + self.fact_registry.isValid(*fact).call().await.map_err(FactCheckerError::FactRegistry)?; + Ok(_0) + } +} diff --git a/crates/prover-services/gps-fact-checker/tests/artifacts/FactRegistry.json b/crates/prover-services/gps-fact-checker/tests/artifacts/FactRegistry.json new file mode 100644 index 00000000..c687ed7e --- /dev/null +++ b/crates/prover-services/gps-fact-checker/tests/artifacts/FactRegistry.json @@ -0,0 +1,863 @@ +{ + "abi": [ + { + "type": "function", + "name": "hasRegisteredFact", + "inputs": [], + "outputs": [{ "name": "", "type": "bool", "internalType": "bool" }], + "stateMutability": "view" + }, + { + "type": "function", + "name": "isValid", + "inputs": [ + { "name": "fact", "type": "bytes32", "internalType": "bytes32" } + ], + "outputs": [{ "name": "", "type": "bool", "internalType": "bool" }], + "stateMutability": "view" + } + ], + "bytecode": { + "object": "0x608060405234801561001057600080fd5b5060ce8061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c80636a938567146037578063d6354e15146065575b600080fd5b605160048036036020811015604b57600080fd5b5035606b565b604080519115158252519081900360200190f35b6051607a565b60006074826083565b92915050565b60015460ff1690565b60009081526020819052604090205460ff169056fea2646970667358221220553e722d7d055d1334a20223ec1ae1a12bf73d8488850f4be28de564102b902764736f6c634300060c0033", + "sourceMap": "118:1279:8:-:0;;;;;;;;;;;;;;;;;;;", + "linkReferences": {} + }, + "deployedBytecode": { + "object": "0x6080604052348015600f57600080fd5b506004361060325760003560e01c80636a938567146037578063d6354e15146065575b600080fd5b605160048036036020811015604b57600080fd5b5035606b565b604080519115158252519081900360200190f35b6051607a565b60006074826083565b92915050565b60015460ff1690565b60009081526020819052604090205460ff169056fea2646970667358221220553e722d7d055d1334a20223ec1ae1a12bf73d8488850f4be28de564102b902764736f6c634300060c0033", + "sourceMap": "118:1279:8:-:0;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;421:109;;;;;;;;;;;;;;;;-1:-1:-1;421:109:8;;:::i;:::-;;;;;;;;;;;;;;;;;;1287:108;;;:::i;421:109::-;484:4;507:16;518:4;507:10;:16::i;:::-;500:23;421:109;-1:-1:-1;;421:109:8:o;1287:108::-;1371:17;;;;1287:108;:::o;826:105::-;883:4;906:18;;;;;;;;;;;;;;826:105::o", + "linkReferences": {} + }, + "methodIdentifiers": { + "hasRegisteredFact()": "d6354e15", + "isValid(bytes32)": "6a938567" + }, + "rawMetadata": "{\"compiler\":{\"version\":\"0.6.12+commit.27d51765\"},\"language\":\"Solidity\",\"output\":{\"abi\":[{\"inputs\":[],\"name\":\"hasRegisteredFact\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"fact\",\"type\":\"bytes32\"}],\"name\":\"isValid\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"}],\"devdoc\":{\"kind\":\"dev\",\"methods\":{},\"version\":1},\"userdoc\":{\"kind\":\"user\",\"methods\":{},\"version\":1}},\"settings\":{\"compilationTarget\":{\"contracts/src/components/FactRegistry.sol\":\"FactRegistry\"},\"evmVersion\":\"istanbul\",\"libraries\":{},\"metadata\":{\"bytecodeHash\":\"ipfs\"},\"optimizer\":{\"enabled\":true,\"runs\":200},\"remappings\":[]},\"sources\":{\"contracts/src/components/FactRegistry.sol\":{\"keccak256\":\"0xf1dde737bfeb616fad002bb7ad229c73fec98f1e539420566fa89805c5bb120d\",\"license\":\"Apache-2.0.\",\"urls\":[\"bzz-raw://1952c89d683c9f58ce06cf222f42772131113b3dd2442c6dc9150a2bde6d4d34\",\"dweb:/ipfs/QmT8u7c1gAYRVF4kCdW7v9QiE65TwwdVu76pdvsFzXnZWg\"]},\"contracts/src/interfaces/IFactRegistry.sol\":{\"keccak256\":\"0xab04b296b506dfb0b4a8828f3dc463072fd50449a5ad8327d1baf01438b0fb35\",\"license\":\"Apache-2.0.\",\"urls\":[\"bzz-raw://e451abe007f98081d1adfd759cc4168f81982992d8c0554650b94d37bc009e64\",\"dweb:/ipfs/QmVBNUWFhNX8PqSMcqRkHVaTbcm7KNpgSg91Sj6MepFG6u\"]},\"contracts/src/interfaces/IQueryableFactRegistry.sol\":{\"keccak256\":\"0x9689f96215bae9da993a5f1b16a7c1460b1abd478569d969d5b901fa4520b4b6\",\"license\":\"Apache-2.0.\",\"urls\":[\"bzz-raw://cfe2f9ca69bffdfaad8cdea188f0e6e385fbd2f5b1ee2194f989f25c76a30250\",\"dweb:/ipfs/QmSffz94BEf9MuqrQ41LuAcBL6FnsxqY4pxfxM8b4s3iSi\"]}},\"version\":1}", + "metadata": { + "compiler": { "version": "0.6.12+commit.27d51765" }, + "language": "Solidity", + "output": { + "abi": [ + { + "inputs": [], + "stateMutability": "view", + "type": "function", + "name": "hasRegisteredFact", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }] + }, + { + "inputs": [ + { "internalType": "bytes32", "name": "fact", "type": "bytes32" } + ], + "stateMutability": "view", + "type": "function", + "name": "isValid", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }] + } + ], + "devdoc": { "kind": "dev", "methods": {}, "version": 1 }, + "userdoc": { "kind": "user", "methods": {}, "version": 1 } + }, + "settings": { + "remappings": [], + "optimizer": { "enabled": true, "runs": 200 }, + "metadata": { "bytecodeHash": "ipfs" }, + "compilationTarget": { + "contracts/src/components/FactRegistry.sol": "FactRegistry" + }, + "libraries": {} + }, + "sources": { + "contracts/src/components/FactRegistry.sol": { + "keccak256": "0xf1dde737bfeb616fad002bb7ad229c73fec98f1e539420566fa89805c5bb120d", + "urls": [ + "bzz-raw://1952c89d683c9f58ce06cf222f42772131113b3dd2442c6dc9150a2bde6d4d34", + "dweb:/ipfs/QmT8u7c1gAYRVF4kCdW7v9QiE65TwwdVu76pdvsFzXnZWg" + ], + "license": "Apache-2.0." + }, + "contracts/src/interfaces/IFactRegistry.sol": { + "keccak256": "0xab04b296b506dfb0b4a8828f3dc463072fd50449a5ad8327d1baf01438b0fb35", + "urls": [ + "bzz-raw://e451abe007f98081d1adfd759cc4168f81982992d8c0554650b94d37bc009e64", + "dweb:/ipfs/QmVBNUWFhNX8PqSMcqRkHVaTbcm7KNpgSg91Sj6MepFG6u" + ], + "license": "Apache-2.0." + }, + "contracts/src/interfaces/IQueryableFactRegistry.sol": { + "keccak256": "0x9689f96215bae9da993a5f1b16a7c1460b1abd478569d969d5b901fa4520b4b6", + "urls": [ + "bzz-raw://cfe2f9ca69bffdfaad8cdea188f0e6e385fbd2f5b1ee2194f989f25c76a30250", + "dweb:/ipfs/QmSffz94BEf9MuqrQ41LuAcBL6FnsxqY4pxfxM8b4s3iSi" + ], + "license": "Apache-2.0." + } + }, + "version": 1 + }, + "ast": { + "absolutePath": "contracts/src/components/FactRegistry.sol", + "id": 1175, + "exportedSymbols": { "FactRegistry": [1174] }, + "nodeType": "SourceUnit", + "src": "40:1358:8", + "nodes": [ + { + "id": 1110, + "nodeType": "PragmaDirective", + "src": "40:24:8", + "nodes": [], + "literals": ["solidity", "^", "0.6", ".12"] + }, + { + "id": 1111, + "nodeType": "ImportDirective", + "src": "66:50:8", + "nodes": [], + "absolutePath": "contracts/src/interfaces/IQueryableFactRegistry.sol", + "file": "../interfaces/IQueryableFactRegistry.sol", + "scope": 1175, + "sourceUnit": 6348, + "symbolAliases": [], + "unitAlias": "" + }, + { + "id": 1174, + "nodeType": "ContractDefinition", + "src": "118:1279:8", + "nodes": [ + { + "id": 1117, + "nodeType": "VariableDeclaration", + "src": "207:45:8", + "nodes": [], + "constant": false, + "mutability": "mutable", + "name": "verifiedFact", + "overrides": null, + "scope": 1174, + "stateVariable": true, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_mapping$_t_bytes32_$_t_bool_$", + "typeString": "mapping(bytes32 => bool)" + }, + "typeName": { + "id": 1116, + "keyType": { + "id": 1114, + "name": "bytes32", + "nodeType": "ElementaryTypeName", + "src": "215:7:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "nodeType": "Mapping", + "src": "207:24:8", + "typeDescriptions": { + "typeIdentifier": "t_mapping$_t_bytes32_$_t_bool_$", + "typeString": "mapping(bytes32 => bool)" + }, + "valueType": { + "id": 1115, + "name": "bool", + "nodeType": "ElementaryTypeName", + "src": "226:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + } + }, + "value": null, + "visibility": "private" + }, + { + "id": 1119, + "nodeType": "VariableDeclaration", + "src": "336:22:8", + "nodes": [], + "constant": false, + "mutability": "mutable", + "name": "anyFactRegistered", + "overrides": null, + "scope": 1174, + "stateVariable": true, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "typeName": { + "id": 1118, + "name": "bool", + "nodeType": "ElementaryTypeName", + "src": "336:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "value": null, + "visibility": "internal" + }, + { + "id": 1132, + "nodeType": "FunctionDefinition", + "src": "421:109:8", + "nodes": [], + "body": { + "id": 1131, + "nodeType": "Block", + "src": "490:40:8", + "nodes": [], + "statements": [ + { + "expression": { + "argumentTypes": null, + "arguments": [ + { + "argumentTypes": null, + "id": 1128, + "name": "fact", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1121, + "src": "518:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + } + ], + "expression": { + "argumentTypes": [ + { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + ], + "id": 1127, + "name": "_factCheck", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1144, + "src": "507:10:8", + "typeDescriptions": { + "typeIdentifier": "t_function_internal_view$_t_bytes32_$returns$_t_bool_$", + "typeString": "function (bytes32) view returns (bool)" + } + }, + "id": 1129, + "isConstant": false, + "isLValue": false, + "isPure": false, + "kind": "functionCall", + "lValueRequested": false, + "names": [], + "nodeType": "FunctionCall", + "src": "507:16:8", + "tryCall": false, + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "functionReturnParameters": 1126, + "id": 1130, + "nodeType": "Return", + "src": "500:23:8" + } + ] + }, + "baseFunctions": [6327], + "documentation": null, + "functionSelector": "6a938567", + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "isValid", + "overrides": { + "id": 1123, + "nodeType": "OverrideSpecifier", + "overrides": [], + "src": "466:8:8" + }, + "parameters": { + "id": 1122, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1121, + "mutability": "mutable", + "name": "fact", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1132, + "src": "438:12:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + }, + "typeName": { + "id": 1120, + "name": "bytes32", + "nodeType": "ElementaryTypeName", + "src": "438:7:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "437:14:8" + }, + "returnParameters": { + "id": 1126, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1125, + "mutability": "mutable", + "name": "", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1132, + "src": "484:4:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "typeName": { + "id": 1124, + "name": "bool", + "nodeType": "ElementaryTypeName", + "src": "484:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "483:6:8" + }, + "scope": 1174, + "stateMutability": "view", + "virtual": false, + "visibility": "external" + }, + { + "id": 1144, + "nodeType": "FunctionDefinition", + "src": "826:105:8", + "nodes": [], + "body": { + "id": 1143, + "nodeType": "Block", + "src": "889:42:8", + "nodes": [], + "statements": [ + { + "expression": { + "argumentTypes": null, + "baseExpression": { + "argumentTypes": null, + "id": 1139, + "name": "verifiedFact", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1117, + "src": "906:12:8", + "typeDescriptions": { + "typeIdentifier": "t_mapping$_t_bytes32_$_t_bool_$", + "typeString": "mapping(bytes32 => bool)" + } + }, + "id": 1141, + "indexExpression": { + "argumentTypes": null, + "id": 1140, + "name": "fact", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1134, + "src": "919:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "isConstant": false, + "isLValue": true, + "isPure": false, + "lValueRequested": false, + "nodeType": "IndexAccess", + "src": "906:18:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "functionReturnParameters": 1138, + "id": 1142, + "nodeType": "Return", + "src": "899:25:8" + } + ] + }, + "documentation": null, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "_factCheck", + "overrides": null, + "parameters": { + "id": 1135, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1134, + "mutability": "mutable", + "name": "fact", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1144, + "src": "846:12:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + }, + "typeName": { + "id": 1133, + "name": "bytes32", + "nodeType": "ElementaryTypeName", + "src": "846:7:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "845:14:8" + }, + "returnParameters": { + "id": 1138, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1137, + "mutability": "mutable", + "name": "", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1144, + "src": "883:4:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "typeName": { + "id": 1136, + "name": "bool", + "nodeType": "ElementaryTypeName", + "src": "883:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "882:6:8" + }, + "scope": 1174, + "stateMutability": "view", + "virtual": false, + "visibility": "internal" + }, + { + "id": 1164, + "nodeType": "FunctionDefinition", + "src": "937:272:8", + "nodes": [], + "body": { + "id": 1163, + "nodeType": "Block", + "src": "986:223:8", + "nodes": [], + "statements": [ + { + "expression": { + "argumentTypes": null, + "id": 1153, + "isConstant": false, + "isLValue": false, + "isPure": false, + "lValueRequested": false, + "leftHandSide": { + "argumentTypes": null, + "baseExpression": { + "argumentTypes": null, + "id": 1149, + "name": "verifiedFact", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1117, + "src": "1058:12:8", + "typeDescriptions": { + "typeIdentifier": "t_mapping$_t_bytes32_$_t_bool_$", + "typeString": "mapping(bytes32 => bool)" + } + }, + "id": 1151, + "indexExpression": { + "argumentTypes": null, + "id": 1150, + "name": "factHash", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1146, + "src": "1071:8:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "isConstant": false, + "isLValue": true, + "isPure": false, + "lValueRequested": true, + "nodeType": "IndexAccess", + "src": "1058:22:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "nodeType": "Assignment", + "operator": "=", + "rightHandSide": { + "argumentTypes": null, + "hexValue": "74727565", + "id": 1152, + "isConstant": false, + "isLValue": false, + "isPure": true, + "kind": "bool", + "lValueRequested": false, + "nodeType": "Literal", + "src": "1083:4:8", + "subdenomination": null, + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "value": "true" + }, + "src": "1058:29:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "id": 1154, + "nodeType": "ExpressionStatement", + "src": "1058:29:8" + }, + { + "condition": { + "argumentTypes": null, + "id": 1156, + "isConstant": false, + "isLValue": false, + "isPure": false, + "lValueRequested": false, + "nodeType": "UnaryOperation", + "operator": "!", + "prefix": true, + "src": "1134:18:8", + "subExpression": { + "argumentTypes": null, + "id": 1155, + "name": "anyFactRegistered", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1119, + "src": "1135:17:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "falseBody": null, + "id": 1162, + "nodeType": "IfStatement", + "src": "1130:73:8", + "trueBody": { + "id": 1161, + "nodeType": "Block", + "src": "1154:49:8", + "statements": [ + { + "expression": { + "argumentTypes": null, + "id": 1159, + "isConstant": false, + "isLValue": false, + "isPure": false, + "lValueRequested": false, + "leftHandSide": { + "argumentTypes": null, + "id": 1157, + "name": "anyFactRegistered", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1119, + "src": "1168:17:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "nodeType": "Assignment", + "operator": "=", + "rightHandSide": { + "argumentTypes": null, + "hexValue": "74727565", + "id": 1158, + "isConstant": false, + "isLValue": false, + "isPure": true, + "kind": "bool", + "lValueRequested": false, + "nodeType": "Literal", + "src": "1188:4:8", + "subdenomination": null, + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "value": "true" + }, + "src": "1168:24:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "id": 1160, + "nodeType": "ExpressionStatement", + "src": "1168:24:8" + } + ] + } + } + ] + }, + "documentation": null, + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "registerFact", + "overrides": null, + "parameters": { + "id": 1147, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1146, + "mutability": "mutable", + "name": "factHash", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1164, + "src": "959:16:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + }, + "typeName": { + "id": 1145, + "name": "bytes32", + "nodeType": "ElementaryTypeName", + "src": "959:7:8", + "typeDescriptions": { + "typeIdentifier": "t_bytes32", + "typeString": "bytes32" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "958:18:8" + }, + "returnParameters": { + "id": 1148, + "nodeType": "ParameterList", + "parameters": [], + "src": "986:0:8" + }, + "scope": 1174, + "stateMutability": "nonpayable", + "virtual": false, + "visibility": "internal" + }, + { + "id": 1173, + "nodeType": "FunctionDefinition", + "src": "1287:108:8", + "nodes": [], + "body": { + "id": 1172, + "nodeType": "Block", + "src": "1354:41:8", + "nodes": [], + "statements": [ + { + "expression": { + "argumentTypes": null, + "id": 1170, + "name": "anyFactRegistered", + "nodeType": "Identifier", + "overloadedDeclarations": [], + "referencedDeclaration": 1119, + "src": "1371:17:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "functionReturnParameters": 1169, + "id": 1171, + "nodeType": "Return", + "src": "1364:24:8" + } + ] + }, + "baseFunctions": [6346], + "documentation": null, + "functionSelector": "d6354e15", + "implemented": true, + "kind": "function", + "modifiers": [], + "name": "hasRegisteredFact", + "overrides": { + "id": 1166, + "nodeType": "OverrideSpecifier", + "overrides": [], + "src": "1330:8:8" + }, + "parameters": { + "id": 1165, + "nodeType": "ParameterList", + "parameters": [], + "src": "1313:2:8" + }, + "returnParameters": { + "id": 1169, + "nodeType": "ParameterList", + "parameters": [ + { + "constant": false, + "id": 1168, + "mutability": "mutable", + "name": "", + "nodeType": "VariableDeclaration", + "overrides": null, + "scope": 1173, + "src": "1348:4:8", + "stateVariable": false, + "storageLocation": "default", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + }, + "typeName": { + "id": 1167, + "name": "bool", + "nodeType": "ElementaryTypeName", + "src": "1348:4:8", + "typeDescriptions": { + "typeIdentifier": "t_bool", + "typeString": "bool" + } + }, + "value": null, + "visibility": "internal" + } + ], + "src": "1347:6:8" + }, + "scope": 1174, + "stateMutability": "view", + "virtual": false, + "visibility": "external" + } + ], + "abstract": false, + "baseContracts": [ + { + "arguments": null, + "baseName": { + "contractScope": null, + "id": 1112, + "name": "IQueryableFactRegistry", + "nodeType": "UserDefinedTypeName", + "referencedDeclaration": 6347, + "src": "143:22:8", + "typeDescriptions": { + "typeIdentifier": "t_contract$_IQueryableFactRegistry_$6347", + "typeString": "contract IQueryableFactRegistry" + } + }, + "id": 1113, + "nodeType": "InheritanceSpecifier", + "src": "143:22:8" + } + ], + "contractDependencies": [6328, 6347], + "contractKind": "contract", + "documentation": null, + "fullyImplemented": true, + "linearizedBaseContracts": [1174, 6347, 6328], + "name": "FactRegistry", + "scope": 1175 + } + ], + "license": "Apache-2.0." + }, + "id": 8 +} diff --git a/crates/prover-services/gps-fact-checker/tests/artifacts/README.md b/crates/prover-services/gps-fact-checker/tests/artifacts/README.md new file mode 100644 index 00000000..a9d4d435 --- /dev/null +++ b/crates/prover-services/gps-fact-checker/tests/artifacts/README.md @@ -0,0 +1,39 @@ +# How to generate artifacts + +## Solidity output for GPS verifier + +Clone and build the repo +using Foundry. + +## Cairo PIEs + +In order to generate zip compressed Cairo PIEs follow the instructions at + + +Few things to note: + +- Use `--cairo_pie_output` flag to specify the output path for the zipped PIE + file +- Use `--append_return_values` flag to write program output to the related + builtin segment +- Use the according layout (that includes `output` builtin at the very least, + so by default `small`) depending on what particular program uses + +Example: + +```sh +cargo run ../cairo_programs/cairo-1-programs/fibonacci.cairo --append_return_values --cairo_pie_output fibonacci.zip --layout small +``` + +### Generate facts + +To create test vectors for SHARP facts you would need to install the Cairo0 +toolchain as described here: + +Then use the `get_fact.py` script to get the fact of the according zipped PIE. + +Example: + +```sh +python3 get_fact.py fibonacci.zip +``` diff --git a/crates/prover-services/gps-fact-checker/tests/artifacts/fibonacci.zip b/crates/prover-services/gps-fact-checker/tests/artifacts/fibonacci.zip new file mode 100644 index 0000000000000000000000000000000000000000..b5943536870ae352374926593e2580178826de0d GIT binary patch literal 1514 zcmWIWW@Zs#U|`^2uyWrNvG>CD@1j7SFc9+qaan3nab|v=URH5_-s&(fzcV_g`t?uv zgl-Blx?&u%ij^S%Mf<`V3to9KGBB7hGcfQ1wdbamB&H;mB!cy?opjpoumg|l|I}-n zKir=8kIPBJ!C_)X)Yg?P4u8w%T`{xGL08rSF+QgPP)+2+`;*38rP{7YmklO^pyyywh`0lJ&@>o@;k3G!ISS zn0M0NW#PJ;kFF+Zm~GB_d8uJ`=G2?oN&(*zq$V~s#LuZKcJj3hx#StjLF=Y(x%l{rD+)Y~`DJ8dglW#hl-OGP_U(UbtF%Nd?UH?+>CVu;0 zi?iXTJ?H=aUizf)$BC{dC%?Y`dpT38>@im-cjlBQ3+KzlebbiEmxx*U_cE92_A8|- z++}lizRsu<*`o2G#?VT3m-NBAe}8hL272cmEx{+iKo{U+VBi7=dTxGErCw5IUhlP& zLAL`0S|4gZ<`g!THkir5siOGV`gA8IxI9Q~Tm74zX6rnDmK20P-BLC8UB$ZF zSLTM5?#{a#w;|8I;$F>%J&qstu-;L4`gw(=etm8Gj-6A^EBU>x#`r;nOq<{XP4aF1+O1o z`M#-t@jmg{$90$GysB#aCsvT8|F!t=iuor6+-vnq|4)@(o|B__6XD8I>7w$eJ`{2s= zx3WMZ-`M({{it~3zWbep1@jAD{roNobkR4~<+BiLlV^fAtqRlf@bTAWGs-HRBINDWWi;8#Tj9^G zD74(YtY~JC575v+AeI2))QZ&PQjqEKMXAO4rA5i9#o&ytd&-dOfC7)h#oZ}o8M{UH z_++RCI0_q{3^=m;`ohQm_b~QkT@iUy*&;f1)faJj%`NxD9(;?@F-{BHZ6T;uX!c5g zy(7B#@O+0m&;I~jz{sS>jJtpWIs*)rG=eDP;tE|Sdf5cg$H1_p(H_V|DXGx4qUSJ# z)-Aw%mzxU9Z|M5aV*{b@0I-0>7B?6spanU?1X&i`5fI?b$_7%&3WTOW7tRN&WncgR DephfO literal 0 HcmV?d00001 diff --git a/crates/prover-services/gps-fact-checker/tests/artifacts/get_fact.py b/crates/prover-services/gps-fact-checker/tests/artifacts/get_fact.py new file mode 100644 index 00000000..3f4e9206 --- /dev/null +++ b/crates/prover-services/gps-fact-checker/tests/artifacts/get_fact.py @@ -0,0 +1,14 @@ +#!/usr/bin/python3 + +import sys +from starkware.cairo.lang.vm.cairo_pie import CairoPie +from starkware.cairo.bootloaders.generate_fact import get_cairo_pie_fact_info +from starkware.cairo.bootloaders.hash_program import compute_program_hash_chain + +cairo_pie = CairoPie.from_file(sys.argv[1]) + +program_hash = compute_program_hash_chain(program=cairo_pie.program, use_poseidon=False) +print("Program hash: ", program_hash) + +fact_info = get_cairo_pie_fact_info(cairo_pie, program_hash) +print("Fact: ", fact_info.fact) diff --git a/crates/prover-services/prover-client-interface/Cargo.toml b/crates/prover-services/prover-client-interface/Cargo.toml new file mode 100644 index 00000000..5f7045c0 --- /dev/null +++ b/crates/prover-services/prover-client-interface/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "prover-client-interface" +version.workspace = true +edition.workspace = true + +[dependencies] +async-trait.workspace = true +cairo-vm.workspace = true +gps-fact-checker.workspace = true +mockall.workspace = true +snos.workspace = true +thiserror.workspace = true +utils.workspace = true diff --git a/crates/prover-services/prover-client-interface/src/lib.rs b/crates/prover-services/prover-client-interface/src/lib.rs new file mode 100644 index 00000000..b8ecdf55 --- /dev/null +++ b/crates/prover-services/prover-client-interface/src/lib.rs @@ -0,0 +1,45 @@ +use async_trait::async_trait; +use cairo_vm::vm::runners::cairo_pie::CairoPie; +use mockall::automock; + +/// Prover client provides an abstraction over different proving services that do the following: +/// - Accept a task containing Cairo intermediate execution artifacts (in PIE format) +/// - Aggregate multiple tasks and prove the execution (of the bootloader program where PIEs are +/// inputs) +/// - Register the proof onchain (individiual proof facts available for each task) +/// +/// A common Madara workflow would be single task per block (SNOS execution result) or per block +/// span (SNAR). +#[automock] +#[async_trait] +pub trait ProverClient: Send + Sync { + async fn submit_task(&self, task: Task) -> Result; + async fn get_task_status(&self, task_id: &TaskId) -> Result; +} + +pub enum Task { + CairoPie(CairoPie), +} + +pub type TaskId = String; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TaskStatus { + Processing, + Succeeded, + Failed(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum ProverClientError { + #[error("Internal prover error: {0}")] + Internal(#[source] Box), + #[error("Settings provider error: {0}")] + SettingsProvider(#[from] utils::settings::SettingsProviderError), + #[error("Task is invalid: {0}")] + TaskInvalid(TaskId), + #[error("Fact checker error: {0}")] + FactChecker(#[from] gps_fact_checker::error::FactCheckerError), + #[error("Failed to encode Cairo PIE: {0}")] + PieEncoding(#[source] snos::error::SnOsError), +} diff --git a/crates/prover-services/sharp-service/Cargo.toml b/crates/prover-services/sharp-service/Cargo.toml new file mode 100644 index 00000000..572ff8b5 --- /dev/null +++ b/crates/prover-services/sharp-service/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "sharp-service" +version.workspace = true +edition.workspace = true + +[dependencies] +alloy.workspace = true +alloy-primitives.workspace = true +async-trait.workspace = true +cairo-vm.workspace = true +gps-fact-checker.workspace = true +hex.workspace = true +prover-client-interface.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json.workspace = true +snos.workspace = true +thiserror.workspace = true +tracing.workspace = true +url.workspace = true +utils.workspace = true +uuid.workspace = true + +[dev-dependencies] +tokio.workspace = true diff --git a/crates/prover-services/sharp-service/src/client.rs b/crates/prover-services/sharp-service/src/client.rs new file mode 100644 index 00000000..e1aa8c14 --- /dev/null +++ b/crates/prover-services/sharp-service/src/client.rs @@ -0,0 +1,49 @@ +use serde_json::json; +use snos::sharp::{CairoJobResponse, CairoStatusResponse}; +use url::Url; +use uuid::Uuid; + +use crate::error::SharpError; + +/// SHARP endpoint for Sepolia testnet +pub const DEFAULT_SHARP_URL: &str = "https://testnet.provingservice.io"; + +/// SHARP API async wrapper +pub struct SharpClient { + base_url: Url, + client: reqwest::Client, +} + +impl SharpClient { + pub fn new(url: Url) -> Self { + Self { base_url: url, client: reqwest::Client::new() } + } + + pub async fn add_job(&self, encoded_pie: &str) -> Result { + let data = json!({ "action": "add_job", "request": { "cairo_pie": encoded_pie } }); + let url = self.base_url.join("add_job").unwrap(); + let res = self.client.post(url).json(&data).send().await.map_err(SharpError::AddJobFailure)?; + + match res.status() { + reqwest::StatusCode::OK => res.json().await.map_err(SharpError::AddJobFailure), + code => Err(SharpError::SharpService(code)), + } + } + + pub async fn get_job_status(&self, job_key: &Uuid) -> Result { + let data = json!({ "action": "get_status", "request": { "cairo_job_key": job_key } }); + let url = self.base_url.join("get_status").unwrap(); + let res = self.client.post(url).json(&data).send().await.map_err(SharpError::GetJobStatusFailure)?; + + match res.status() { + reqwest::StatusCode::OK => res.json().await.map_err(SharpError::GetJobStatusFailure), + code => Err(SharpError::SharpService(code)), + } + } +} + +impl Default for SharpClient { + fn default() -> Self { + Self::new(DEFAULT_SHARP_URL.parse().unwrap()) + } +} diff --git a/crates/prover-services/sharp-service/src/config.rs b/crates/prover-services/sharp-service/src/config.rs new file mode 100644 index 00000000..6567448c --- /dev/null +++ b/crates/prover-services/sharp-service/src/config.rs @@ -0,0 +1,27 @@ +use alloy::primitives::Address; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::client::DEFAULT_SHARP_URL; + +/// SHARP proving service configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SharpConfig { + /// SHARP service url + pub service_url: Url, + /// EVM RPC node url + pub rpc_node_url: Url, + /// GPS verifier contract address (implements FactRegistry) + pub verifier_address: Address, +} + +impl Default for SharpConfig { + /// Default config for Sepolia testnet + fn default() -> Self { + Self { + service_url: DEFAULT_SHARP_URL.parse().unwrap(), + rpc_node_url: "https://sepolia.drpc.org".parse().unwrap(), + verifier_address: "0x07ec0D28e50322Eb0C159B9090ecF3aeA8346DFe".parse().unwrap(), + } + } +} diff --git a/crates/prover-services/sharp-service/src/error.rs b/crates/prover-services/sharp-service/src/error.rs new file mode 100644 index 00000000..082d6690 --- /dev/null +++ b/crates/prover-services/sharp-service/src/error.rs @@ -0,0 +1,30 @@ +use alloy_primitives::hex::FromHexError; +use gps_fact_checker::error::FactCheckerError; +use prover_client_interface::ProverClientError; +use reqwest::StatusCode; + +#[derive(Debug, thiserror::Error)] +pub enum SharpError { + #[error("Failed to to add SHARP job: {0}")] + AddJobFailure(#[source] reqwest::Error), + #[error("Failed to to get status of a SHARP job: {0}")] + GetJobStatusFailure(#[source] reqwest::Error), + #[error("Fact checker error: {0}")] + FactChecker(#[from] FactCheckerError), + #[error("SHARP service returned an error {0}")] + SharpService(StatusCode), + #[error("Failed to parse job key: {0}")] + JobKeyParse(uuid::Error), + #[error("Failed to parse fact: {0}")] + FactParse(FromHexError), + #[error("Failed to split task id into job key and fact")] + TaskIdSplit, + #[error("Failed to encode PIE")] + PieEncode(#[source] snos::error::SnOsError), +} + +impl From for ProverClientError { + fn from(value: SharpError) -> Self { + Self::Internal(Box::new(value)) + } +} diff --git a/crates/prover-services/sharp-service/src/lib.rs b/crates/prover-services/sharp-service/src/lib.rs new file mode 100644 index 00000000..800fc835 --- /dev/null +++ b/crates/prover-services/sharp-service/src/lib.rs @@ -0,0 +1,140 @@ +pub mod client; +pub mod config; +pub mod error; + +use std::str::FromStr; + +use alloy::primitives::B256; +use async_trait::async_trait; +use gps_fact_checker::fact_info::get_fact_info; +use gps_fact_checker::FactChecker; +use prover_client_interface::{ProverClient, ProverClientError, Task, TaskId, TaskStatus}; +use snos::sharp::CairoJobStatus; +use utils::settings::SettingsProvider; +use uuid::Uuid; + +use crate::client::SharpClient; +use crate::config::SharpConfig; +use crate::error::SharpError; + +pub const SHARP_SETTINGS_NAME: &str = "sharp"; + +/// SHARP (aka GPS) is a shared proving service hosted by Starkware. +pub struct SharpProverService { + sharp_client: SharpClient, + fact_checker: FactChecker, +} + +#[async_trait] +impl ProverClient for SharpProverService { + async fn submit_task(&self, task: Task) -> Result { + match task { + Task::CairoPie(cairo_pie) => { + let fact_info = get_fact_info(&cairo_pie, None)?; + let encoded_pie = + snos::sharp::pie::encode_pie_mem(cairo_pie).map_err(ProverClientError::PieEncoding)?; + let res = self.sharp_client.add_job(&encoded_pie).await?; + if let Some(job_key) = res.cairo_job_key { + Ok(combine_task_id(&job_key, &fact_info.fact)) + } else { + Err(ProverClientError::TaskInvalid(res.error_message.unwrap_or_default())) + } + } + } + } + + async fn get_task_status(&self, task_id: &TaskId) -> Result { + let (job_key, fact) = split_task_id(task_id)?; + let res = self.sharp_client.get_job_status(&job_key).await?; + match res.status { + CairoJobStatus::FAILED => Ok(TaskStatus::Failed(res.error_log.unwrap_or_default())), + CairoJobStatus::INVALID => { + Ok(TaskStatus::Failed(format!("Task is invalid: {:?}", res.invalid_reason.unwrap_or_default()))) + } + CairoJobStatus::UNKNOWN => Ok(TaskStatus::Failed(format!("Task not found: {}", task_id))), + CairoJobStatus::IN_PROGRESS | CairoJobStatus::NOT_CREATED | CairoJobStatus::PROCESSED => { + Ok(TaskStatus::Processing) + } + CairoJobStatus::ONCHAIN => { + if self.fact_checker.is_valid(&fact).await? { + Ok(TaskStatus::Succeeded) + } else { + Ok(TaskStatus::Failed(format!("Fact {} is not valid or not registed", hex::encode(fact)))) + } + } + } + } +} + +impl SharpProverService { + pub fn new(sharp_client: SharpClient, fact_checker: FactChecker) -> Self { + Self { sharp_client, fact_checker } + } + + pub fn with_settings(settings: &impl SettingsProvider) -> Self { + let sharp_cfg: SharpConfig = settings.get_settings(SHARP_SETTINGS_NAME).unwrap(); + let sharp_client = SharpClient::new(sharp_cfg.service_url); + let fact_checker = FactChecker::new(sharp_cfg.rpc_node_url, sharp_cfg.verifier_address); + Self::new(sharp_client, fact_checker) + } +} + +/// Construct SHARP specific task ID from job key and proof fact +pub fn combine_task_id(job_key: &Uuid, fact: &B256) -> TaskId { + format!("{}:{}", job_key, fact) +} + +/// Split task ID into SHARP job key and proof fact +pub fn split_task_id(task_id: &TaskId) -> Result<(Uuid, B256), SharpError> { + let (job_key_str, fact_str) = task_id.split_once(':').ok_or(SharpError::TaskIdSplit)?; + let job_key = Uuid::from_str(job_key_str).map_err(SharpError::JobKeyParse)?; + let fact = B256::from_str(fact_str).map_err(SharpError::FactParse)?; + Ok((job_key, fact)) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::time::Duration; + + use cairo_vm::vm::runners::cairo_pie::CairoPie; + use prover_client_interface::{ProverClient, Task, TaskStatus}; + use tracing::log::log; + use tracing::log::Level::{Error, Info}; + use utils::settings::default::DefaultSettingsProvider; + + use crate::SharpProverService; + + #[ignore] + #[tokio::test] + async fn sharp_reproduce_rate_limiting_issue() { + // TODO: leaving this test to check if the issue still reproduces (504 error after 8th job status + // query) + let sharp_service = SharpProverService::with_settings(&DefaultSettingsProvider {}); + let cairo_pie_path: PathBuf = + [env!("CARGO_MANIFEST_DIR"), "tests", "artifacts", "fibonacci.zip"].iter().collect(); + let cairo_pie = CairoPie::read_zip_file(&cairo_pie_path).unwrap(); + // Submit task to the testnet prover + let task_id = sharp_service.submit_task(Task::CairoPie(cairo_pie)).await.unwrap(); + log!(Info, "SHARP: task {} submitted", task_id); + for attempt in 0..10 { + tokio::time::sleep(Duration::from_millis((attempt + 1) * 1000)).await; + match sharp_service.get_task_status(&task_id).await.unwrap() { + TaskStatus::Failed(err) => { + log!(Error, "SHARP: task failed with {}", err); + panic!("Task failed"); + } + TaskStatus::Processing => { + log!(Info, "SHARP: task is processing (attempt {})", attempt); + continue; + } + TaskStatus::Succeeded => { + log!(Info, "SHARP: task is completed"); + return; + } + } + } + log!(Error, "SHARP: waiting timeout"); + panic!("Out of attempts"); + } +} diff --git a/crates/prover-services/sharp-service/tests/artifacts/fibonacci.zip b/crates/prover-services/sharp-service/tests/artifacts/fibonacci.zip new file mode 100644 index 0000000000000000000000000000000000000000..b5943536870ae352374926593e2580178826de0d GIT binary patch literal 1514 zcmWIWW@Zs#U|`^2uyWrNvG>CD@1j7SFc9+qaan3nab|v=URH5_-s&(fzcV_g`t?uv zgl-Blx?&u%ij^S%Mf<`V3to9KGBB7hGcfQ1wdbamB&H;mB!cy?opjpoumg|l|I}-n zKir=8kIPBJ!C_)X)Yg?P4u8w%T`{xGL08rSF+QgPP)+2+`;*38rP{7YmklO^pyyywh`0lJ&@>o@;k3G!ISS zn0M0NW#PJ;kFF+Zm~GB_d8uJ`=G2?oN&(*zq$V~s#LuZKcJj3hx#StjLF=Y(x%l{rD+)Y~`DJ8dglW#hl-OGP_U(UbtF%Nd?UH?+>CVu;0 zi?iXTJ?H=aUizf)$BC{dC%?Y`dpT38>@im-cjlBQ3+KzlebbiEmxx*U_cE92_A8|- z++}lizRsu<*`o2G#?VT3m-NBAe}8hL272cmEx{+iKo{U+VBi7=dTxGErCw5IUhlP& zLAL`0S|4gZ<`g!THkir5siOGV`gA8IxI9Q~Tm74zX6rnDmK20P-BLC8UB$ZF zSLTM5?#{a#w;|8I;$F>%J&qstu-;L4`gw(=etm8Gj-6A^EBU>x#`r;nOq<{XP4aF1+O1o z`M#-t@jmg{$90$GysB#aCsvT8|F!t=iuor6+-vnq|4)@(o|B__6XD8I>7w$eJ`{2s= zx3WMZ-`M({{it~3zWbep1@jAD{roNobkR4~<+BiLlV^fAtqRlf@bTAWGs-HRBINDWWi;8#Tj9^G zD74(YtY~JC575v+AeI2))QZ&PQjqEKMXAO4rA5i9#o&ytd&-dOfC7)h#oZ}o8M{UH z_++RCI0_q{3^=m;`ohQm_b~QkT@iUy*&;f1)faJj%`NxD9(;?@F-{BHZ6T;uX!c5g zy(7B#@O+0m&;I~jz{sS>jJtpWIs*)rG=eDP;tE|Sdf5cg$H1_p(H_V|DXGx4qUSJ# z)-Aw%mzxU9Z|M5aV*{b@0I-0>7B?6spanU?1X&i`5fI?b$_7%&3WTOW7tRN&WncgR DephfO literal 0 HcmV?d00001 diff --git a/crates/settlement_clients/settlement-client-interface/Cargo.toml b/crates/settlement-clients/settlement-client-interface/Cargo.toml similarity index 100% rename from crates/settlement_clients/settlement-client-interface/Cargo.toml rename to crates/settlement-clients/settlement-client-interface/Cargo.toml diff --git a/crates/settlement_clients/settlement-client-interface/src/lib.rs b/crates/settlement-clients/settlement-client-interface/src/lib.rs similarity index 96% rename from crates/settlement_clients/settlement-client-interface/src/lib.rs rename to crates/settlement-clients/settlement-client-interface/src/lib.rs index da5a471c..7fc51a65 100644 --- a/crates/settlement_clients/settlement-client-interface/src/lib.rs +++ b/crates/settlement-clients/settlement-client-interface/src/lib.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use color_eyre::Result; -use mockall::{automock, predicate::*}; +use mockall::automock; +use mockall::predicate::*; use starknet::core::types::FieldElement; #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 8fea4db4..1061637d 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -7,3 +7,5 @@ edition.workspace = true [dependencies] color-eyre = { workspace = true } +serde.workspace = true +thiserror.workspace = true diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 6a65fdce..277ce5ab 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -1 +1,14 @@ pub mod env_utils; +pub mod settings; + +/// Evaluate `$x:expr` and if not true return `Err($y:expr)`. +/// +/// Used as `ensure!(expression_to_ensure, expression_to_return_on_false)`. +#[macro_export] +macro_rules! ensure { + ($x:expr, $y:expr $(,)?) => {{ + if !$x { + return Err($y); + } + }}; +} diff --git a/crates/utils/src/settings/default.rs b/crates/utils/src/settings/default.rs new file mode 100644 index 00000000..3166ab6e --- /dev/null +++ b/crates/utils/src/settings/default.rs @@ -0,0 +1,13 @@ +use super::SettingsProvider; + +#[derive(Debug, Clone, Default)] +pub struct DefaultSettingsProvider {} + +impl SettingsProvider for DefaultSettingsProvider { + fn get_settings( + &self, + _section: &'static str, + ) -> Result { + Ok(T::default()) + } +} diff --git a/crates/utils/src/settings/mod.rs b/crates/utils/src/settings/mod.rs new file mode 100644 index 00000000..cbf87c43 --- /dev/null +++ b/crates/utils/src/settings/mod.rs @@ -0,0 +1,13 @@ +pub mod default; + +use serde::de::DeserializeOwned; + +#[derive(Debug, thiserror::Error)] +pub enum SettingsProviderError { + #[error("Internal settings error: {0}")] + Internal(#[source] Box), +} + +pub trait SettingsProvider { + fn get_settings(&self, name: &'static str) -> Result; +} diff --git a/e2e-tests/Cargo.toml b/e2e-tests/Cargo.toml new file mode 100644 index 00000000..03ffa215 --- /dev/null +++ b/e2e-tests/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "e2e-tests" +version = "0.1.0" +edition = "2021" + +[dependencies] +orchestrator.workspace = true +reqwest = { workspace = true, features = ["json"] } +serde_json.workspace = true +testcontainers.workspace = true +tokio = { workspace = true, features = ["full"] } +tokio-stream.workspace = true +tokio-util.workspace = true +url.workspace = true + +[[test]] +name = "test_samples" +path = "test_samples.rs" diff --git a/e2e-tests/src/lib.rs b/e2e-tests/src/lib.rs new file mode 100644 index 00000000..4e4b2ae1 --- /dev/null +++ b/e2e-tests/src/lib.rs @@ -0,0 +1,28 @@ +pub mod mongodb; +pub mod node; + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; + +pub use mongodb::MongoDbServer; +pub use node::Orchestrator; +pub use orchestrator::database::mongodb::MongoDb as MongoDbClient; + +const MIN_PORT: u16 = 49_152; +const MAX_PORT: u16 = 65_535; + +fn get_free_port() -> u16 { + for port in MIN_PORT..=MAX_PORT { + if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) { + return listener.local_addr().expect("No local addr").port(); + } + // otherwise port is occupied + } + panic!("No free ports available"); +} + +fn get_repository_root() -> PathBuf { + let manifest_path = Path::new(&env!("CARGO_MANIFEST_DIR")); + let repository_root = manifest_path.parent().expect("Failed to get parent directory of CARGO_MANIFEST_DIR"); + repository_root.to_path_buf() +} diff --git a/e2e-tests/src/mongodb.rs b/e2e-tests/src/mongodb.rs new file mode 100644 index 00000000..0d99020f --- /dev/null +++ b/e2e-tests/src/mongodb.rs @@ -0,0 +1,34 @@ +use testcontainers::core::{ContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{ContainerAsync, GenericImage, ImageExt}; +use url::Url; + +use crate::get_free_port; + +const MONGODB_DEFAULT_PORT: u16 = 27017; +const MONGODB_IMAGE_NAME: &str = "mongo"; +const MONGODB_IMAGE_TAG: &str = "8.0-rc"; + +#[allow(dead_code)] +pub struct MongoDbServer { + container: ContainerAsync, + endpoint: Url, +} + +impl MongoDbServer { + pub async fn run() -> Self { + let host_port = get_free_port(); + + let container = GenericImage::new(MONGODB_IMAGE_NAME, MONGODB_IMAGE_TAG) + .with_wait_for(WaitFor::message_on_stdout("Waiting for connections")) + .with_mapped_port(host_port, ContainerPort::Tcp(MONGODB_DEFAULT_PORT)) + .start() + .await + .expect("Failed to create docker container"); + Self { container, endpoint: Url::parse(&format!("http://127.0.0.1:{}", host_port)).unwrap() } + } + + pub fn endpoint(&self) -> &Url { + &self.endpoint + } +} diff --git a/e2e-tests/src/node.rs b/e2e-tests/src/node.rs new file mode 100644 index 00000000..3a00d67a --- /dev/null +++ b/e2e-tests/src/node.rs @@ -0,0 +1,89 @@ +use std::fs::{create_dir_all, File}; +use std::path::Path; +use std::process::{Child, Command, ExitStatus, Stdio}; +use std::time::Duration; + +use tokio::net::TcpStream; +use url::Url; + +use crate::{get_free_port, get_repository_root}; + +const CONNECTION_ATTEMPTS: usize = 360; +const CONNECTION_ATTEMPT_DELAY_MS: u64 = 500; + +#[derive(Debug)] +pub struct Orchestrator { + process: Child, + address: String, +} + +impl Drop for Orchestrator { + fn drop(&mut self) { + let mut kill = + Command::new("kill").args(["-s", "TERM", &self.process.id().to_string()]).spawn().expect("Failed to kill"); + kill.wait().expect("Failed to kill the process"); + } +} + +impl Orchestrator { + fn cargo_run(root_dir: &Path, binary: &str, args: Vec<&str>, envs: Vec<(&str, &str)>) -> Child { + let arguments = [vec!["run", "--bin", binary, "--release", "--"], args].concat(); + + let logs_dir = Path::join(root_dir, Path::new("target/logs")); + create_dir_all(logs_dir.clone()).expect("Failed to create logs dir"); + + let stdout = Stdio::from(File::create(logs_dir.join(format!("{}-stdout.txt", binary))).unwrap()); + let stderr = Stdio::from(File::create(logs_dir.join(format!("{}-stderr.txt", binary))).unwrap()); + + Command::new("cargo") + .stdout(stdout) + .stderr(stderr) + .envs(envs) + .args(arguments) + .spawn() + .expect("Could not run orchestrator node") + } + + pub fn run(envs: Vec<(&str, &str)>) -> Self { + let port = get_free_port(); + let address = format!("127.0.0.1:{}", port); + let repository_root = &get_repository_root(); + + std::env::set_current_dir(repository_root).expect("Failed to change working directory"); + + let port_str = format!("{}", port); + let envs = [envs, vec![("PORT", port_str.as_str())]].concat(); + + let process = Self::cargo_run(repository_root.as_path(), "orchestrator", vec![], envs); + + Self { process, address } + } + + pub fn endpoint(&self) -> Url { + Url::parse(&format!("http://{}", self.address)).unwrap() + } + + pub fn has_exited(&mut self) -> Option { + self.process.try_wait().expect("Failed to get orchestrator node exit status") + } + + pub async fn wait_till_started(&mut self) { + let mut attempts = CONNECTION_ATTEMPTS; + loop { + match TcpStream::connect(&self.address).await { + Ok(_) => return, + Err(err) => { + if let Some(status) = self.has_exited() { + panic!("Orchestrator node exited early with {}", status); + } + if attempts == 0 { + panic!("Failed to connect to {}: {}", self.address, err); + } + } + }; + + attempts -= 1; + tokio::time::sleep(Duration::from_millis(CONNECTION_ATTEMPT_DELAY_MS)).await; + } + } +} diff --git a/e2e-tests/test_samples.rs b/e2e-tests/test_samples.rs new file mode 100644 index 00000000..d593fa08 --- /dev/null +++ b/e2e-tests/test_samples.rs @@ -0,0 +1,15 @@ +use e2e_tests::{MongoDbServer, Orchestrator}; + +extern crate e2e_tests; + +#[ignore = "requires DOCKER_HOST set to run"] +#[tokio::test] +async fn test_orchestrator_launches() { + let mongodb = MongoDbServer::run().await; + let mut orchestrator = Orchestrator::run(vec![ + // TODO: mock Madara RPC API + ("MADARA_RPC_URL", "http://localhost"), + ("MONGODB_CONNECTION_STRING", mongodb.endpoint().as_str()), + ]); + orchestrator.wait_till_started().await; +} diff --git a/migrations/.DS_Store b/migrations/.DS_Store deleted file mode 100644 index 60e51e56ddd77652cc36abb4b383072dd075bf78..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeH~y-EW?5XWcrgrH3t!OjI=z=L40JHz<|0UL`mF~JWmUcjWXx!$L;u(Ghy$~W*O z{LjwlB_tM+_<_tEv;X~?x$F=2HcLdVHSWbkbt0<4S)+9nbBxE?XKY1#c+iC$yHq_h z)2+h@7=b@VfcNgs#ICLO?|Uty9$iqMuHiXls9(WH@J?uR|A@vk%+KQXX)zee3fJzI zmw8a6X*Vyr$XI(kzIl0ixIb8C_TD#UAD&NA@%Y)aNij9Se4qC4G^v4h17t6$pe{IP z;MqYfr~WikceGgaB)8k`?Dbi=E%5$9QL*`2hB@nJ24=rc0B1H^Z9~*bBVYuKzy|^T zK6p55qFRcEPX~%{1%O8A4u(44B{;^bCaR?uelhezG z(>FW4p}6pNXxF~I>Ei6gT)|tie tS?l2^a5kP-iiQvrdK`xeAH`enU>NgUfhMY@D0*P_M<8IZ!U+5*fj8WRff4`!