feat: allow to override provider url upon installation (#346)

Allow to override the URL that will be queried by the EVM RPC canister
via a regex specified upon canister installation. This is allows testing
against a local Ethereum development setup (such as
[foundry](https://github.com/foundry-rs/foundry)) without changing the
client calling the EVM RPC canister.

.github/workflows/ci.yml

@@ -92,7 +92,16 @@ jobs:
         run: scripts/e2e
 
       - name: Run examples
-        run: scripts/examples
+        run: scripts/examples evm_rpc 'Number = 20000000'
+
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+
+      - name: Run anvil
+        run: anvil &
+
+      - name: Run local examples with Foundry
+        run: scripts/examples evm_rpc_local 'Number = 0'
 
       - name: Check formatting
         run: cargo fmt --all -- --check

candid/evm_rpc.did

@@ -107,6 +107,7 @@ type InstallArgs = record {
   demo : opt bool;
   manageApiKeys : opt vec principal;
   logFilter : opt LogFilter;
+  overrideProvider : opt OverrideProvider;
 };
 type Regex = text;
 type LogFilter = variant {
@@ -115,6 +116,15 @@ type LogFilter = variant {
   ShowPattern : Regex;
   HidePattern : Regex;
 };
+type RegexSubstitution = record {
+  pattern : Regex;
+  replacement: text;
+};
+// Override resolved provider.
+// Useful for testing with a local Ethereum developer environment such as foundry.
+type OverrideProvider = record {
+  overrideUrl : opt RegexSubstitution
+};
 type JsonRpcError = record { code : int64; message : text };
 type LogEntry = record {
   transactionHash : opt text;

dfx.json

@@ -24,6 +24,13 @@
       "gzip": true,
       "init_arg": "(record {demo = opt true})"
     },
+    "evm_rpc_local": {
+      "candid": "candid/evm_rpc.did",
+      "type": "rust",
+      "package": "evm_rpc",
+      "gzip": true,
+      "init_arg": "( record {  overrideProvider = opt record { overrideUrl = opt record { pattern = \".*\"; replacement = \"http://127.0.0.1:8545\" } } })"
+    },
     "evm_rpc_staging": {
       "candid": "candid/evm_rpc.did",
       "type": "rust",

evm_rpc_types/src/lib.rs

@@ -15,7 +15,7 @@ use serde::{Deserialize, Serialize};
 use std::fmt::{Debug, Display, Formatter};
 use std::str::FromStr;
 
-pub use lifecycle::{InstallArgs, LogFilter, RegexString};
+pub use lifecycle::{InstallArgs, LogFilter, OverrideProvider, RegexString, RegexSubstitution};
 pub use request::{
     AccessList, AccessListEntry, BlockTag, CallArgs, FeeHistoryArgs, GetLogsArgs,
     GetTransactionCountArgs, TransactionRequest,

evm_rpc_types/src/lifecycle/mod.rs

@@ -8,6 +8,8 @@ pub struct InstallArgs {
     pub manage_api_keys: Option<Vec<Principal>>,
     #[serde(rename = "logFilter")]
     pub log_filter: Option<LogFilter>,
+    #[serde(rename = "overrideProvider")]
+    pub override_provider: Option<OverrideProvider>,
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)]
@@ -18,5 +20,17 @@ pub enum LogFilter {
     HidePattern(RegexString),
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Eq, CandidType, Serialize, Deserialize)]
+pub struct OverrideProvider {
+    #[serde(rename = "overrideUrl")]
+    pub override_url: Option<RegexSubstitution>,
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)]
 pub struct RegexString(pub String);
+
+#[derive(Clone, Debug, PartialEq, Eq, CandidType, Serialize, Deserialize)]
+pub struct RegexSubstitution {
+    pub pattern: RegexString,
+    pub replacement: String,
+}

scripts/e2e

@@ -4,6 +4,7 @@
     dfx canister create --all &&
     npm run generate &&
     dfx deploy evm_rpc --mode reinstall -y &&
+    dfx deploy evm_rpc_local --mode reinstall -y &&
     dfx deploy evm_rpc_demo --mode reinstall -y &&
     dfx deploy evm_rpc_staging --mode reinstall -y &&
     dfx deploy e2e_rust &&

scripts/examples

@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 # Run a variety of example RPC calls.
 
+CANISTER_ID=${1:-evm_rpc}
+# Use concrete block height to avoid flakiness on CI
+BLOCK_HEIGHT=${2:-'Number = 20000000'}
+
 NETWORK=local
 IDENTITY=default
-CANISTER_ID=evm_rpc
 CYCLES=10000000000
 WALLET=$(dfx identity get-wallet --network=$NETWORK --identity=$IDENTITY)
 RPC_SERVICE="EthMainnet=variant {PublicNode}"
 RPC_SERVICES=EthMainnet
 RPC_CONFIG="opt record {responseConsensus = opt variant {Threshold = record {total = opt (3 : nat8); min = 2 : nat8}}}"
-# Use concrete block height to avoid flakiness on CI
-BLOCK_HEIGHT="Number = 20000000"
 
 FLAGS="--network=$NETWORK --identity=$IDENTITY --with-cycles=$CYCLES --wallet=$WALLET"
 

src/http.rs

@@ -2,7 +2,7 @@ use crate::{
     accounting::{get_cost_with_collateral, get_http_request_cost},
     add_metric_entry,
     constants::{CONTENT_TYPE_HEADER_LOWERCASE, CONTENT_TYPE_VALUE},
-    memory::is_demo_active,
+    memory::{get_override_provider, is_demo_active},
     types::{MetricRpcHost, MetricRpcMethod, ResolvedRpcService},
     util::canonicalize_json,
 };
@@ -20,7 +20,7 @@ pub async fn json_rpc_request(
     max_response_bytes: u64,
 ) -> RpcResult<HttpResponse> {
     let cycles_cost = get_http_request_cost(json_rpc_payload.len() as u64, max_response_bytes);
-    let api = service.api();
+    let api = service.api(&get_override_provider())?;
     let mut request_headers = api.headers.unwrap_or_default();
     if !request_headers
         .iter()
@@ -42,28 +42,27 @@ pub async fn json_rpc_request(
             vec![],
         )),
     };
-    http_request(rpc_method, service, request, cycles_cost).await
+    http_request(rpc_method, request, cycles_cost).await
 }
 
 pub async fn http_request(
     rpc_method: MetricRpcMethod,
-    service: ResolvedRpcService,
     request: CanisterHttpRequestArgument,
     cycles_cost: u128,
 ) -> RpcResult<HttpResponse> {
-    let api = service.api();
-    let parsed_url = match url::Url::parse(&api.url) {
+    let url = request.url.clone();
+    let parsed_url = match url::Url::parse(&url) {
         Ok(url) => url,
         Err(_) => {
-            return Err(ValidationError::Custom(format!("Error parsing URL: {}", api.url)).into())
+            return Err(ValidationError::Custom(format!("Error parsing URL: {}", url)).into())
         }
     };
     let host = match parsed_url.host_str() {
         Some(host) => host,
         None => {
             return Err(ValidationError::Custom(format!(
                 "Error parsing hostname from URL: {}",
-                api.url
+                url
             ))
             .into())
         }

src/main.rs

@@ -6,11 +6,11 @@ use evm_rpc::http::get_http_response_body;
 use evm_rpc::logs::INFO;
 use evm_rpc::memory::{
     insert_api_key, is_api_key_principal, is_demo_active, remove_api_key, set_api_key_principals,
-    set_demo_active, set_log_filter,
+    set_demo_active, set_log_filter, set_override_provider,
 };
 use evm_rpc::metrics::encode_metrics;
 use evm_rpc::providers::{find_provider, resolve_rpc_service, PROVIDERS, SERVICE_PROVIDER_MAP};
-use evm_rpc::types::{LogFilter, Provider, ProviderId, RpcAccess, RpcAuth};
+use evm_rpc::types::{LogFilter, OverrideProvider, Provider, ProviderId, RpcAccess, RpcAuth};
 use evm_rpc::{
     http::{json_rpc_request, transform_http_request},
     http_types,
@@ -269,7 +269,13 @@ fn post_upgrade(args: evm_rpc_types::InstallArgs) {
         set_api_key_principals(principals);
     }
     if let Some(filter) = args.log_filter {
-        set_log_filter(LogFilter::from(filter))
+        set_log_filter(LogFilter::try_from(filter).expect("ERROR: Invalid log filter"));
+    }
+    if let Some(override_provider) = args.override_provider {
+        set_override_provider(
+            OverrideProvider::try_from(override_provider)
+                .expect("ERROR: invalid override provider"),
+        );
     }
 }
 

src/memory.rs

@@ -7,12 +7,13 @@ use ic_stable_structures::{
 use ic_stable_structures::{Cell, StableBTreeMap};
 use std::cell::RefCell;
 
-use crate::types::{ApiKey, LogFilter, Metrics, ProviderId};
+use crate::types::{ApiKey, LogFilter, Metrics, OverrideProvider, ProviderId};
 
 const IS_DEMO_ACTIVE_MEMORY_ID: MemoryId = MemoryId::new(4);
 const API_KEY_MAP_MEMORY_ID: MemoryId = MemoryId::new(5);
 const MANAGE_API_KEYS_MEMORY_ID: MemoryId = MemoryId::new(6);
 const LOG_FILTER_MEMORY_ID: MemoryId = MemoryId::new(7);
+const OVERRIDE_PROVIDER_MEMORY_ID: MemoryId = MemoryId::new(8);
 
 type StableMemory = VirtualMemory<DefaultMemoryImpl>;
 
@@ -31,7 +32,9 @@ thread_local! {
     static MANAGE_API_KEYS: RefCell<ic_stable_structures::Vec<Principal, StableMemory>> =
         RefCell::new(ic_stable_structures::Vec::init(MEMORY_MANAGER.with_borrow(|m| m.get(MANAGE_API_KEYS_MEMORY_ID))).expect("Unable to read API key principals from stable memory"));
     static LOG_FILTER: RefCell<Cell<LogFilter, StableMemory>> =
-        RefCell::new(ic_stable_structures::Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(LOG_FILTER_MEMORY_ID)), LogFilter::default()).expect("Unable to read log message filter from stable memory"));
+        RefCell::new(Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(LOG_FILTER_MEMORY_ID)), LogFilter::default()).expect("Unable to read log message filter from stable memory"));
+    static OVERRIDE_PROVIDER: RefCell<Cell<OverrideProvider, StableMemory>> =
+        RefCell::new(Cell::init(MEMORY_MANAGER.with_borrow(|m| m.get(OVERRIDE_PROVIDER_MEMORY_ID)), OverrideProvider::default()).expect("Unable to read provider override from stable memory"));
 }
 
 pub fn get_api_key(provider_id: ProviderId) -> Option<ApiKey> {
@@ -86,6 +89,18 @@ pub fn set_log_filter(filter: LogFilter) {
     });
 }
 
+pub fn get_override_provider() -> OverrideProvider {
+    OVERRIDE_PROVIDER.with_borrow(|provider| provider.get().clone())
+}
+
+pub fn set_override_provider(provider: OverrideProvider) {
+    OVERRIDE_PROVIDER.with_borrow_mut(|state| {
+        state
+            .set(provider)
+            .expect("Error while updating override provider")
+    });
+}
+
 pub fn next_request_id() -> u64 {
     UNSTABLE_HTTP_REQUEST_COUNTER.with_borrow_mut(|counter| {
         let current_request_id = *counter;

src/rpc_client/eth_rpc/mod.rs

@@ -3,17 +3,17 @@
 
 use crate::accounting::get_http_request_cost;
 use crate::logs::{DEBUG, TRACE_HTTP};
-use crate::memory::next_request_id;
+use crate::memory::{get_override_provider, next_request_id};
 use crate::providers::resolve_rpc_service;
 use crate::rpc_client::eth_rpc_error::{sanitize_send_raw_transaction_result, Parser};
 use crate::rpc_client::json::requests::JsonRpcRequest;
 use crate::rpc_client::json::responses::{
     Block, FeeHistory, JsonRpcReply, JsonRpcResult, LogEntry, TransactionReceipt,
 };
 use crate::rpc_client::numeric::{TransactionCount, Wei};
-use crate::types::MetricRpcMethod;
+use crate::types::{MetricRpcMethod, OverrideProvider};
 use candid::candid_method;
-use evm_rpc_types::{HttpOutcallError, ProviderError, RpcApi, RpcError, RpcService};
+use evm_rpc_types::{HttpOutcallError, RpcApi, RpcError, RpcService};
 use ic_canister_log::log;
 use ic_cdk::api::call::RejectionCode;
 use ic_cdk::api::management_canister::http_request::{
@@ -181,7 +181,7 @@ where
         method: eth_method.clone(),
         id: 1,
     };
-    let api = resolve_api(provider)?;
+    let api = resolve_api(provider, &get_override_provider())?;
     let url = &api.url;
     let mut headers = vec![HttpHeader {
         name: "Content-Type".to_string(),
@@ -221,9 +221,7 @@ where
             )),
         };
 
-        let response = match http_request(provider, &eth_method, request, effective_size_estimate)
-            .await
-        {
+        let response = match http_request(&eth_method, request, effective_size_estimate).await {
             Err(RpcError::HttpOutcallError(HttpOutcallError::IcError { code, message }))
                 if is_response_too_large(&code, &message) =>
             {
@@ -273,17 +271,18 @@ where
     }
 }
 
-fn resolve_api(service: &RpcService) -> Result<RpcApi, ProviderError> {
-    Ok(resolve_rpc_service(service.clone())?.api())
+fn resolve_api(
+    service: &RpcService,
+    override_provider: &OverrideProvider,
+) -> Result<RpcApi, RpcError> {
+    resolve_rpc_service(service.clone())?.api(override_provider)
 }
 
 async fn http_request(
-    service: &RpcService,
     method: &str,
     request: CanisterHttpRequestArgument,
     effective_response_size_estimate: u64,
 ) -> Result<HttpResponse, RpcError> {
-    let service = resolve_rpc_service(service.clone())?;
     let cycles_cost = get_http_request_cost(
         request
             .body
@@ -293,7 +292,7 @@ async fn http_request(
         effective_response_size_estimate,
     );
     let rpc_method = MetricRpcMethod(method.to_string());
-    crate::http::http_request(rpc_method, service, request, cycles_cost).await
+    crate::http::http_request(rpc_method, request, cycles_cost).await
 }
 
 fn http_status_code(response: &HttpResponse) -> u16 {