diff --git a/README.md b/README.md index aefb7cf..01c6794 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ This is a generalized JSON RPC proxy server with features specifically designed Pull vendors: `git submodule update --init --recursive` -Quick start: `cargo run -- --config config.yml` +Quick start: `cargo run -- --config configs/config.yml` -This will run a proxy server with [config.yml](config.yml) as the configuration file. +This will run a proxy server with [config.yml](configs/config.yml) as the configuration file. Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://github.com/tokio-rs/console) @@ -24,7 +24,11 @@ Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://gi - Log format. Default: `full`. - Options: `full`, `pretty`, `json`, `compact` -In addition, you can refer env variables in `config.yml` by using `${SOME_ENV}` +In addition, you can refer env variables in `config.yml` by using following syntax: + +- `${variable}` +- `${variable:-word}` indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. +- `${variable:+word}` indicates that if variable is set then word will be the result, otherwise the result is the empty string. ## Features diff --git a/config.yml b/configs/config.yml similarity index 100% rename from config.yml rename to configs/config.yml diff --git a/configs/config_with_env.yml b/configs/config_with_env.yml new file mode 100644 index 0000000..ff70c97 --- /dev/null +++ b/configs/config_with_env.yml @@ -0,0 +1,56 @@ +extensions: + client: + endpoints: + - wss://acala-rpc.dwellir.com + - wss://acala-rpc-0.aca-api.network + health_check: + interval_sec: 10 # check interval, default is 10s + healthy_response_time_ms: 500 # max response time to be considered healthy, default is 500ms + health_method: system_health + response: # response contains { isSyncing: false } + !contains + - - isSyncing + - !eq false + event_bus: + substrate_api: + stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes + telemetry: + provider: none + cache: + default_ttl_seconds: 60 + default_size: 500 + merge_subscription: + keep_alive_seconds: 60 + server: + port: ${SUBWAY_PORT:-9944} + listen_address: '0.0.0.0' + max_connections: ${SUBWAY_MAX_CONNECTIONS:-2000} + http_methods: + - path: /health + method: system_health + - path: /liveness + method: chain_getBlockHash + cors: all + rate_limit: # these are for demo purpose only, please adjust to your needs + connection: # 20 RPC requests per second per connection + burst: 20 + period_secs: 1 + ip: # 500 RPC requests per 10 seconds per ip + burst: 500 + period_secs: 10 + # use X-Forwarded-For header to get real ip, if available (e.g. behind a load balancer). + # WARNING: Use with caution, as this xff header can be forged. + use_xff: true # default is false + +middlewares: + methods: + - delay + - response + - inject_params + - cache + - upstream + subscriptions: + - merge_subscription + - upstream + +rpcs: substrate diff --git a/eth_config.yml b/configs/eth_config.yml similarity index 100% rename from eth_config.yml rename to configs/eth_config.yml diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..4e32bbb --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::path::PathBuf; +#[derive(Parser, Debug)] +#[command(version, about)] +pub struct Command { + /// The config file to use + #[arg(short, long, default_value = ".configs/config.yml")] + pub config: PathBuf, +} +pub fn parse_args() -> Command { + Command::parse() +} diff --git a/src/config/mod.rs b/src/config/mod.rs index 41fd6fa..7366721 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,8 +2,8 @@ use anyhow::{bail, Context}; use regex::{Captures, Regex}; use std::env; use std::fs; +use std::path; -use clap::Parser; use serde::Deserialize; use crate::extensions::ExtensionsConfig; @@ -14,14 +14,6 @@ mod rpc; const SUBSTRATE_CONFIG: &str = include_str!("../../rpc_configs/substrate.yml"); const ETHEREUM_CONFIG: &str = include_str!("../../rpc_configs/ethereum.yml"); -#[derive(Parser, Debug)] -#[command(version, about)] -struct Command { - /// The config file to use - #[arg(short, long, default_value = "./config.yml")] - config: String, -} - #[derive(Deserialize, Debug)] pub struct RpcDefinitionsWithBase { #[serde(default)] @@ -150,17 +142,16 @@ impl From for Config { } // read config file specified in command line -pub fn read_config() -> Result { - let cmd = Command::parse(); - +pub fn read_config(path: impl AsRef) -> Result { + let path = path.as_ref(); let templated_config_str = - fs::read_to_string(&cmd.config).with_context(|| format!("Unable to read config file: {}", cmd.config))?; + fs::read_to_string(path).with_context(|| format!("Unable to read config file: {}", path.display()))?; let config_str = render_template(&templated_config_str) - .with_context(|| format!("Unable to preprocess config file: {}", cmd.config))?; + .with_context(|| format!("Unable to preprocess config file: {}", path.display()))?; - let config: ParseConfig = - serde_yaml::from_str(&config_str).with_context(|| format!("Unable to parse config file: {}", cmd.config))?; + let config: ParseConfig = serde_yaml::from_str(&config_str) + .with_context(|| format!("Unable to parse config file: {}", path.display()))?; let config: Config = config.into(); // TODO: shouldn't need to do this here. Creating a server should validates everything @@ -170,18 +161,37 @@ pub fn read_config() -> Result { } fn render_template(templated_config_str: &str) -> Result { - // match pattern: ${SOME_VAR} - let re = Regex::new(r"\$\{([^\}]+)\}").unwrap(); + // match pattern with 1 group: {variable_name} + // match pattern with 3 groups: {variable:-word} or {variable:+word} + // note: incompete syntax like {variable:-} will be matched since group1 is ungreedy match + // but typically it will be rejected due to there is not corresponding env vars + let re = Regex::new(r"\$\{([^}]+?)(?:(:-|:\+)([^}]+))?\}").unwrap(); let mut config_str = String::with_capacity(templated_config_str.len()); let mut last_match = 0; // replace pattern: with env variables - let replacement = |caps: &Captures| -> Result { env::var(&caps[1]) }; + let replacement = |caps: &Captures| -> Result { + match (caps.get(2), caps.get(3)) { + (Some(sign), Some(value_default)) => { + if sign.as_str() == ":-" { + env::var(&caps[1]).or(Ok(value_default.as_str().to_string())) + } else if sign.as_str() == ":+" { + Ok(env::var(&caps[1]).map_or("".to_string(), |_| value_default.as_str().to_string())) + } else { + Err(env::VarError::NotPresent) + } + } + (None, None) => env::var(&caps[1]), + _ => Err(env::VarError::NotPresent), + } + }; // replace every matches with early return // when encountering error for caps in re.captures_iter(templated_config_str) { - let m = caps.get(0).expect("Matched pattern should have at least one capture"); + let m = caps + .get(0) + .expect("i==0 means implicit unnamed group that includes the entire match, which is infalliable"); config_str.push_str(&templated_config_str[last_match..m.start()]); config_str.push_str( &replacement(&caps).with_context(|| format!("Unable to replace environment variable {}", &caps[1]))?, @@ -231,13 +241,56 @@ mod tests { fn render_template_basically_works() { env::set_var("KEY", "value"); env::set_var("ANOTHER_KEY", "another_value"); - let templated_config_str = "${KEY} ${ANOTHER_KEY}"; + let templated_config_str = "${KEY} some random_$tring {inside ${ANOTHER_KEY}"; let config_str = render_template(templated_config_str).unwrap(); - assert_eq!(config_str, "value another_value"); + assert_eq!(config_str, "value some random_$tring {inside another_value"); env::remove_var("KEY"); let config_str = render_template(templated_config_str); assert!(config_str.is_err()); env::remove_var("ANOTHER_KEY"); } + + #[test] + fn render_template_supports_minus_word_syntax() { + // ${variable:-word} indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. + env::set_var("absent_key", "value_set"); + let templated_config_str = "${absent_key:-value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_set"); + // remove the env + env::remove_var("absent_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default") + } + + #[test] + fn render_template_supports_plus_word_syntax() { + // ${variable:+word} indicates that if variable is set then word will be the result, otherwise the result is the empty string. + env::set_var("present_key", "any_value"); + let templated_config_str = "${present_key:+value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default"); + // remove the env + env::remove_var("present_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "") + } + + #[test] + fn render_template_gets_error_when_syntax_is_incomplete() { + let templated_config_str = "${variable:-}"; + let config_str = render_template(templated_config_str); + assert!(config_str.is_err()); + let template_config_str = "${variable:+}"; + let config_str = render_template(template_config_str); + assert!(config_str.is_err()); + } + + #[test] + fn read_config_with_render_template_works() { + // It's enough to check the replacement works + // if config itself has proper data validation + let _config = read_config("configs/config_with_env.yml").unwrap(); + } } diff --git a/src/lib.rs b/src/lib.rs index 7e244e3..55bdcc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod cli; pub mod config; pub mod extensions; pub mod logger; diff --git a/src/main.rs b/src/main.rs index 26da8b4..046ee2d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { // read config from file - let config = subway::config::read_config()?; + let cli = subway::cli::parse_args(); + let config = subway::config::read_config(&cli.config)?; subway::logger::enable_logger(); tracing::trace!("{:#?}", config);