From be8158ffef5023d601c958e682ba65ac4abb2c10 Mon Sep 17 00:00:00 2001 From: indirection42 Date: Tue, 23 Apr 2024 10:03:30 +0800 Subject: [PATCH 1/4] feat(config): support ${var:-word} and ${var:+word} syntax --- config.yml => configs/config.yml | 0 configs/config_with_env.yml | 56 ++++++++++++++ eth_config.yml => configs/eth_config.yml | 0 src/cli.rs | 12 +++ src/config/mod.rs | 96 ++++++++++++++++++------ src/lib.rs | 1 + src/main.rs | 3 +- 7 files changed, 145 insertions(+), 23 deletions(-) rename config.yml => configs/config.yml (100%) create mode 100644 configs/config_with_env.yml rename eth_config.yml => configs/eth_config.yml (100%) create mode 100644 src/cli.rs diff --git a/config.yml b/configs/config.yml similarity index 100% rename from config.yml rename to configs/config.yml diff --git a/configs/config_with_env.yml b/configs/config_with_env.yml new file mode 100644 index 0000000..ff70c97 --- /dev/null +++ b/configs/config_with_env.yml @@ -0,0 +1,56 @@ +extensions: + client: + endpoints: + - wss://acala-rpc.dwellir.com + - wss://acala-rpc-0.aca-api.network + health_check: + interval_sec: 10 # check interval, default is 10s + healthy_response_time_ms: 500 # max response time to be considered healthy, default is 500ms + health_method: system_health + response: # response contains { isSyncing: false } + !contains + - - isSyncing + - !eq false + event_bus: + substrate_api: + stale_timeout_seconds: 180 # rotate endpoint if no new blocks for 3 minutes + telemetry: + provider: none + cache: + default_ttl_seconds: 60 + default_size: 500 + merge_subscription: + keep_alive_seconds: 60 + server: + port: ${SUBWAY_PORT:-9944} + listen_address: '0.0.0.0' + max_connections: ${SUBWAY_MAX_CONNECTIONS:-2000} + http_methods: + - path: /health + method: system_health + - path: /liveness + method: chain_getBlockHash + cors: all + rate_limit: # these are for demo purpose only, please adjust to your needs + connection: # 20 RPC requests per second per connection + burst: 20 + period_secs: 1 + ip: # 500 RPC requests per 10 seconds per ip + burst: 500 + period_secs: 10 + # use X-Forwarded-For header to get real ip, if available (e.g. behind a load balancer). + # WARNING: Use with caution, as this xff header can be forged. + use_xff: true # default is false + +middlewares: + methods: + - delay + - response + - inject_params + - cache + - upstream + subscriptions: + - merge_subscription + - upstream + +rpcs: substrate diff --git a/eth_config.yml b/configs/eth_config.yml similarity index 100% rename from eth_config.yml rename to configs/eth_config.yml diff --git a/src/cli.rs b/src/cli.rs new file mode 100644 index 0000000..4e32bbb --- /dev/null +++ b/src/cli.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use std::path::PathBuf; +#[derive(Parser, Debug)] +#[command(version, about)] +pub struct Command { + /// The config file to use + #[arg(short, long, default_value = ".configs/config.yml")] + pub config: PathBuf, +} +pub fn parse_args() -> Command { + Command::parse() +} diff --git a/src/config/mod.rs b/src/config/mod.rs index 41fd6fa..1e47a5f 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -2,8 +2,8 @@ use anyhow::{bail, Context}; use regex::{Captures, Regex}; use std::env; use std::fs; +use std::path; -use clap::Parser; use serde::Deserialize; use crate::extensions::ExtensionsConfig; @@ -14,14 +14,6 @@ mod rpc; const SUBSTRATE_CONFIG: &str = include_str!("../../rpc_configs/substrate.yml"); const ETHEREUM_CONFIG: &str = include_str!("../../rpc_configs/ethereum.yml"); -#[derive(Parser, Debug)] -#[command(version, about)] -struct Command { - /// The config file to use - #[arg(short, long, default_value = "./config.yml")] - config: String, -} - #[derive(Deserialize, Debug)] pub struct RpcDefinitionsWithBase { #[serde(default)] @@ -150,17 +142,16 @@ impl From for Config { } // read config file specified in command line -pub fn read_config() -> Result { - let cmd = Command::parse(); - +pub fn read_config(path: impl AsRef) -> Result { + let path = path.as_ref(); let templated_config_str = - fs::read_to_string(&cmd.config).with_context(|| format!("Unable to read config file: {}", cmd.config))?; + fs::read_to_string(path).with_context(|| format!("Unable to read config file: {}", path.display()))?; let config_str = render_template(&templated_config_str) - .with_context(|| format!("Unable to preprocess config file: {}", cmd.config))?; + .with_context(|| format!("Unable to preprocess config file: {}", path.display()))?; - let config: ParseConfig = - serde_yaml::from_str(&config_str).with_context(|| format!("Unable to parse config file: {}", cmd.config))?; + let config: ParseConfig = serde_yaml::from_str(&config_str) + .with_context(|| format!("Unable to parse config file: {}", path.display()))?; let config: Config = config.into(); // TODO: shouldn't need to do this here. Creating a server should validates everything @@ -170,18 +161,35 @@ pub fn read_config() -> Result { } fn render_template(templated_config_str: &str) -> Result { - // match pattern: ${SOME_VAR} - let re = Regex::new(r"\$\{([^\}]+)\}").unwrap(); + // match pattern: ${SOME_VAR}, ${SOME_VAR:-word}, or ${SOME_VAR:+word} + // TODO: Partial syntax like ${SOME_VAR:-} and ${SOME_VAR:+} should be invalid, but it's not supported yet + let re = Regex::new(r"\$\{([^\}:-]+)(?:(:-|:\+)([^\}]*))?\}").unwrap(); let mut config_str = String::with_capacity(templated_config_str.len()); let mut last_match = 0; // replace pattern: with env variables - let replacement = |caps: &Captures| -> Result { env::var(&caps[1]) }; + let replacement = |caps: &Captures| -> Result { + match (caps.get(2), caps.get(3)) { + (Some(sign), Some(value_default)) => { + if sign.as_str() == ":-" { + env::var(&caps[1]).or(Ok(value_default.as_str().to_string())) + } else if sign.as_str() == ":+" { + Ok(env::var(&caps[1]).map_or("".to_string(), |_| value_default.as_str().to_string())) + } else { + Err(env::VarError::NotPresent) + } + } + (None, None) => env::var(&caps[1]), + _ => Err(env::VarError::NotPresent), + } + }; // replace every matches with early return // when encountering error for caps in re.captures_iter(templated_config_str) { - let m = caps.get(0).expect("Matched pattern should have at least one capture"); + let m = caps + .get(0) + .expect("i==0 means implicit unnamed group that includes the entire match, which is infalliable"); config_str.push_str(&templated_config_str[last_match..m.start()]); config_str.push_str( &replacement(&caps).with_context(|| format!("Unable to replace environment variable {}", &caps[1]))?, @@ -231,13 +239,57 @@ mod tests { fn render_template_basically_works() { env::set_var("KEY", "value"); env::set_var("ANOTHER_KEY", "another_value"); - let templated_config_str = "${KEY} ${ANOTHER_KEY}"; + let templated_config_str = "${KEY} some random_$tring {inside ${ANOTHER_KEY}"; let config_str = render_template(templated_config_str).unwrap(); - assert_eq!(config_str, "value another_value"); + assert_eq!(config_str, "value some random_$tring {inside another_value"); env::remove_var("KEY"); let config_str = render_template(templated_config_str); assert!(config_str.is_err()); env::remove_var("ANOTHER_KEY"); } + + #[test] + fn render_template_supports_minus_word_syntax() { + // ${variable:-word} indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. + env::set_var("absent_key", "value_set"); + let templated_config_str = "${absent_key:-value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_set"); + // remove the env + env::remove_var("absent_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default") + } + + #[test] + fn render_template_supports_plus_word_syntax() { + // ${variable:+word} indicates that if variable is set then word will be the result, otherwise the result is the empty string. + env::set_var("present_key", "any_value"); + let templated_config_str = "${present_key:+value_default}"; + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "value_default"); + // remove the env + env::remove_var("present_key"); + let config_str = render_template(templated_config_str).unwrap(); + assert_eq!(config_str, "") + } + + #[test] + #[ignore = "not supported yet"] + fn render_template_gets_error_when_syntax_is_partial() { + let templated_config_str = "${variable:-}"; + let config_str = render_template(templated_config_str); + assert!(config_str.is_err()); + let template_config_str = "${variable:+}"; + let config_str = render_template(template_config_str); + assert!(config_str.is_err()); + } + + #[test] + fn read_config_with_render_template_works() { + // It's enough to check the replacement works + // if config itself has proper data validation + let _config = read_config("configs/config_with_env.yml").unwrap(); + } } diff --git a/src/lib.rs b/src/lib.rs index 7e244e3..55bdcc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +pub mod cli; pub mod config; pub mod extensions; pub mod logger; diff --git a/src/main.rs b/src/main.rs index 26da8b4..046ee2d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,8 @@ #[tokio::main] async fn main() -> anyhow::Result<()> { // read config from file - let config = subway::config::read_config()?; + let cli = subway::cli::parse_args(); + let config = subway::config::read_config(&cli.config)?; subway::logger::enable_logger(); tracing::trace!("{:#?}", config); From 1d1a56154d4e2f3fffb643dd991c1d4dc19ff01b Mon Sep 17 00:00:00 2001 From: indirection42 Date: Tue, 23 Apr 2024 10:52:39 +0800 Subject: [PATCH 2/4] feat(config): support rejects incomplete syntax like ${var:-} --- src/config/mod.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/config/mod.rs b/src/config/mod.rs index 1e47a5f..7366721 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -161,9 +161,11 @@ pub fn read_config(path: impl AsRef) -> Result Result { - // match pattern: ${SOME_VAR}, ${SOME_VAR:-word}, or ${SOME_VAR:+word} - // TODO: Partial syntax like ${SOME_VAR:-} and ${SOME_VAR:+} should be invalid, but it's not supported yet - let re = Regex::new(r"\$\{([^\}:-]+)(?:(:-|:\+)([^\}]*))?\}").unwrap(); + // match pattern with 1 group: {variable_name} + // match pattern with 3 groups: {variable:-word} or {variable:+word} + // note: incompete syntax like {variable:-} will be matched since group1 is ungreedy match + // but typically it will be rejected due to there is not corresponding env vars + let re = Regex::new(r"\$\{([^}]+?)(?:(:-|:\+)([^}]+))?\}").unwrap(); let mut config_str = String::with_capacity(templated_config_str.len()); let mut last_match = 0; @@ -276,8 +278,7 @@ mod tests { } #[test] - #[ignore = "not supported yet"] - fn render_template_gets_error_when_syntax_is_partial() { + fn render_template_gets_error_when_syntax_is_incomplete() { let templated_config_str = "${variable:-}"; let config_str = render_template(templated_config_str); assert!(config_str.is_err()); From fc8c2b24d4a3d736a957abb8f9257a4d13f4e87f Mon Sep 17 00:00:00 2001 From: indirection42 Date: Tue, 23 Apr 2024 11:16:06 +0800 Subject: [PATCH 3/4] docs(config): update env handling descriptions --- README.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index aefb7cf..5b38d6a 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ This is a generalized JSON RPC proxy server with features specifically designed Pull vendors: `git submodule update --init --recursive` -Quick start: `cargo run -- --config config.yml` +Quick start: `cargo run -- --config configs/config.yml` -This will run a proxy server with [config.yml](config.yml) as the configuration file. +This will run a proxy server with [config.yml](configs/config.yml) as the configuration file. Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://github.com/tokio-rs/console) @@ -24,7 +24,11 @@ Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://gi - Log format. Default: `full`. - Options: `full`, `pretty`, `json`, `compact` -In addition, you can refer env variables in `config.yml` by using `${SOME_ENV}` +In addition, you can refer env variables in `config.yml` by using following syntax: + +- `${variable}` +- `${variable:-word}` indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. or +- `${variable:+word}` indicates that if variable is set then word will be the result, otherwise the result is the empty string. ## Features From 850d2e6ac2a24c4ff002f30144e7f9fb20766af4 Mon Sep 17 00:00:00 2001 From: indirection42 Date: Tue, 23 Apr 2024 11:17:12 +0800 Subject: [PATCH 4/4] docs(config): update env handling descriptions --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b38d6a..01c6794 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Run with `RUSTFLAGS="--cfg tokio_unstable"` to enable [tokio-console](https://gi In addition, you can refer env variables in `config.yml` by using following syntax: - `${variable}` -- `${variable:-word}` indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. or +- `${variable:-word}` indicates that if variable is set then the result will be that value. If variable is not set then word will be the result. - `${variable:+word}` indicates that if variable is set then word will be the result, otherwise the result is the empty string. ## Features