Enhancement : Refactor Settings

CHANGELOG.md

@@ -31,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
 ## Changed
 
+- settings provider
 - refactor AWS config usage and clean .env files
 - GitHub's coverage CI yml file for localstack and db testing.
 - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder.

crates/da-clients/ethereum/src/config.rs

@@ -1,34 +1,20 @@
-use std::str::FromStr;
-
-use alloy::{network::Ethereum, providers::ProviderBuilder, rpc::client::RpcClient};
-use async_trait::async_trait;
-use da_client_interface::DaConfig;
-use url::Url;
+use serde::{Deserialize, Serialize};
 use utils::env_utils::get_env_var_or_panic;
 
-use crate::EthereumDaClient;
-
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct EthereumDaConfig {
     pub rpc_url: String,
     pub memory_pages_contract: String,
     pub private_key: String,
 }
 
-#[async_trait]
-impl DaConfig<EthereumDaClient> for EthereumDaConfig {
-    fn new_from_env() -> Self {
+impl Default for EthereumDaConfig {
+    /// Default config for Sepolia testnet
+    fn default() -> Self {
         Self {
             rpc_url: get_env_var_or_panic("SETTLEMENT_RPC_URL"),
             memory_pages_contract: get_env_var_or_panic("MEMORY_PAGES_CONTRACT_ADDRESS"),
             private_key: get_env_var_or_panic("PRIVATE_KEY"),
         }
     }
-    async fn build_client(&self) -> EthereumDaClient {
-        let client =
-            RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
-        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
-
-        EthereumDaClient { provider }
-    }
 }

crates/da-clients/ethereum/src/lib.rs

@@ -1,15 +1,23 @@
 #![allow(missing_docs)]
 #![allow(clippy::missing_docs_in_private_items)]
 
+use crate::config::EthereumDaConfig;
 use alloy::network::Ethereum;
-use alloy::providers::RootProvider;
+use alloy::providers::{ProviderBuilder, RootProvider};
+use alloy::rpc::client::RpcClient;
 use alloy::transports::http::Http;
 use async_trait::async_trait;
 use color_eyre::Result;
 use da_client_interface::{DaClient, DaVerificationStatus};
 use mockall::automock;
 use mockall::predicate::*;
 use reqwest::Client;
+use std::str::FromStr;
+use url::Url;
+use utils::settings::SettingsProvider;
+
+pub const DA_SETTINGS_NAME: &str = "ethereum";
+
 pub mod config;
 pub struct EthereumDaClient {
     #[allow(dead_code)]
@@ -37,3 +45,13 @@ impl DaClient for EthereumDaClient {
         131072
     }
 }
+
+impl EthereumDaClient {
+    pub fn with_settings(settings: &impl SettingsProvider) -> Self {
+        let config: EthereumDaConfig = settings.get_settings(DA_SETTINGS_NAME).unwrap();
+        let client =
+            RpcClient::new_http(Url::from_str(config.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL"));
+        let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client);
+        EthereumDaClient { provider }
+    }
+}

crates/orchestrator/src/alerts/aws_sns/config.rs

@@ -0,0 +1,16 @@
+use serde::{Deserialize, Serialize};
+use utils::env_utils::get_env_var_or_panic;
+
+#[derive(Clone, Serialize, Deserialize)]
+pub struct AWSSNSConfig {
+    /// AWS SNS ARN
+    pub sns_arn: String,
+    /// AWS SNS region
+    pub sns_arn_region: String,
+}
+
+impl Default for AWSSNSConfig {
+    fn default() -> Self {
+        Self { sns_arn: get_env_var_or_panic("AWS_SNS_ARN"), sns_arn_region: get_env_var_or_panic("AWS_SNS_REGION") }
+    }
+}

crates/orchestrator/src/alerts/aws_sns/mod.rs

@@ -1,27 +1,31 @@
+mod config;
+
+use crate::alerts::aws_sns::config::AWSSNSConfig;
 use crate::alerts::Alerts;
 use async_trait::async_trait;
 use aws_sdk_sns::config::Region;
 use aws_sdk_sns::Client;
-use utils::env_utils::get_env_var_or_panic;
+use utils::settings::SettingsProvider;
+
+pub const AWS_SNS_SETTINGS_NAME: &str = "sns";
 
 pub struct AWSSNS {
     client: Client,
+    topic_arn: String,
 }
 
 impl AWSSNS {
-    /// To create a new SNS client
-    pub async fn new() -> Self {
-        let sns_region = get_env_var_or_panic("AWS_SNS_REGION");
-        let config = aws_config::from_env().region(Region::new(sns_region)).load().await;
-        AWSSNS { client: Client::new(&config) }
+    pub async fn with_settings(settings: &impl SettingsProvider) -> Self {
+        let sns_config: AWSSNSConfig = settings.get_settings(AWS_SNS_SETTINGS_NAME).unwrap();
+        let config = aws_config::from_env().region(Region::new(sns_config.sns_arn_region)).load().await;
+        Self { client: Client::new(&config), topic_arn: sns_config.sns_arn }
     }
 }
 
 #[async_trait]
 impl Alerts for AWSSNS {
     async fn send_alert_message(&self, message_body: String) -> color_eyre::Result<()> {
-        let topic_arn = get_env_var_or_panic("AWS_SNS_ARN");
-        self.client.publish().topic_arn(topic_arn).message(message_body).send().await?;
+        self.client.publish().topic_arn(self.topic_arn.clone()).message(message_body).send().await?;
         Ok(())
     }
 }

crates/orchestrator/src/config.rs

@@ -2,14 +2,13 @@ use std::sync::Arc;
 
 use crate::alerts::aws_sns::AWSSNS;
 use crate::alerts::Alerts;
-use crate::data_storage::aws_s3::config::AWSS3Config;
 use crate::data_storage::aws_s3::AWSS3;
-use crate::data_storage::{DataStorage, DataStorageConfig};
+use crate::data_storage::DataStorage;
 use arc_swap::{ArcSwap, Guard};
 use aws_config::SdkConfig;
-use da_client_interface::{DaClient, DaConfig};
+use da_client_interface::DaClient;
 use dotenvy::dotenv;
-use ethereum_da_client::config::EthereumDaConfig;
+use ethereum_da_client::EthereumDaClient;
 use ethereum_settlement_client::EthereumSettlementClient;
 use prover_client_interface::ProverClient;
 use settlement_client_interface::SettlementClient;
@@ -22,9 +21,8 @@ use utils::env_utils::get_env_var_or_panic;
 use utils::settings::default::DefaultSettingsProvider;
 use utils::settings::SettingsProvider;
 
-use crate::database::mongodb::config::MongoDbConfig;
 use crate::database::mongodb::MongoDb;
-use crate::database::{Database, DatabaseConfig};
+use crate::database::Database;
 use crate::queue::sqs::SqsQueue;
 use crate::queue::QueueProvider;
 
@@ -58,9 +56,6 @@ pub async fn init_config() -> Config {
         Url::parse(get_env_var_or_panic("MADARA_RPC_URL").as_str()).expect("Failed to parse URL"),
     ));
 
-    // init database
-    let database = build_database_client().await;
-
     // init AWS
     let aws_config = aws_config::load_from_env().await;
 
@@ -70,15 +65,14 @@ pub async fn init_config() -> Config {
     // us stop using the generic omniqueue abstractions for message ack/nack
     let queue = build_queue_client(&aws_config);
 
-    let da_client = build_da_client().await;
-
     let settings_provider = DefaultSettingsProvider {};
+    // init database
+    let database = build_database_client(&settings_provider).await;
+    let da_client = build_da_client(&settings_provider).await;
     let settlement_client = build_settlement_client(&settings_provider).await;
     let prover_client = build_prover_service(&settings_provider);
-
-    let storage_client = build_storage_client(&aws_config).await;
-
-    let alerts_client = build_alert_client().await;
+    let storage_client = build_storage_client(&settings_provider).await;
+    let alerts_client = build_alert_client(&settings_provider).await;
 
     Config::new(
         Arc::new(provider),
@@ -177,12 +171,9 @@ pub async fn config_force_init(config: Config) {
 }
 
 /// Builds the DA client based on the environment variable DA_LAYER
-pub async fn build_da_client() -> Box<dyn DaClient + Send + Sync> {
+pub async fn build_da_client(settings_provider: &impl SettingsProvider) -> Box<dyn DaClient + Send + Sync> {
     match get_env_var_or_panic("DA_LAYER").as_str() {
-        "ethereum" => {
-            let config = EthereumDaConfig::new_from_env();
-            Box::new(config.build_client().await)
-        }
+        "ethereum" => Box::new(EthereumDaClient::with_settings(settings_provider)),
         _ => panic!("Unsupported DA layer"),
     }
 }
@@ -206,29 +197,30 @@ pub async fn build_settlement_client(
     }
 }
 
-pub async fn build_storage_client(aws_config: &SdkConfig) -> Box<dyn DataStorage + Send + Sync> {
+pub async fn build_storage_client(settings_provider: &impl SettingsProvider) -> Box<dyn DataStorage + Send + Sync> {
     match get_env_var_or_panic("DATA_STORAGE").as_str() {
-        "s3" => Box::new(AWSS3::new(AWSS3Config::new_from_env(), aws_config)),
+        "s3" => Box::new(AWSS3::with_settings(settings_provider).await),
         _ => panic!("Unsupported Storage Client"),
     }
 }
 
-pub async fn build_alert_client() -> Box<dyn Alerts + Send + Sync> {
+pub async fn build_alert_client(settings_provider: &impl SettingsProvider) -> Box<dyn Alerts + Send + Sync> {
     match get_env_var_or_panic("ALERTS").as_str() {
-        "sns" => Box::new(AWSSNS::new().await),
+        "sns" => Box::new(AWSSNS::with_settings(settings_provider).await),
         _ => panic!("Unsupported Alert Client"),
     }
 }
+
 pub fn build_queue_client(_aws_config: &SdkConfig) -> Box<dyn QueueProvider + Send + Sync> {
     match get_env_var_or_panic("QUEUE_PROVIDER").as_str() {
         "sqs" => Box::new(SqsQueue {}),
         _ => panic!("Unsupported Queue Client"),
     }
 }
 
-pub async fn build_database_client() -> Box<dyn Database + Send + Sync> {
+pub async fn build_database_client(settings_provider: &impl SettingsProvider) -> Box<dyn Database + Send + Sync> {
     match get_env_var_or_panic("DATABASE").as_str() {
-        "mongodb" => Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await),
+        "mongodb" => Box::new(MongoDb::with_settings(settings_provider).await),
         _ => panic!("Unsupported Database Client"),
     }
 }

crates/orchestrator/src/data_storage/aws_s3/config.rs

@@ -1,9 +1,10 @@
+use serde::{Deserialize, Serialize};
 use utils::env_utils::get_env_var_or_panic;
 
 use crate::data_storage::DataStorageConfig;
 
 /// Represents AWS S3 config struct with all the necessary variables.
-#[derive(Clone)]
+#[derive(Clone, Serialize, Deserialize)]
 pub struct AWSS3Config {
     /// S3 Bucket Name
     pub bucket_name: String,
@@ -16,3 +17,10 @@ impl DataStorageConfig for AWSS3Config {
         Self { bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME") }
     }
 }
+
+impl Default for AWSS3Config {
+    /// Default config for aws s3
+    fn default() -> Self {
+        Self { bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME") }
+    }
+}

crates/orchestrator/src/data_storage/aws_s3/mod.rs

@@ -1,11 +1,13 @@
 use crate::data_storage::aws_s3::config::AWSS3Config;
 use crate::data_storage::DataStorage;
 use async_trait::async_trait;
-use aws_config::SdkConfig;
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::Client;
 use bytes::Bytes;
 use color_eyre::Result;
+use utils::settings::SettingsProvider;
+
+pub const S3_SETTINGS_NAME: &str = "s3";
 
 /// Module for AWS S3 config structs and implementations
 pub mod config;
@@ -20,11 +22,13 @@ pub struct AWSS3 {
 ///
 /// - initializing a new AWS S3 client
 impl AWSS3 {
-    /// Initializes a new AWS S3 client by passing the config
-    /// and returning it.
-    pub fn new(s3_config: AWSS3Config, aws_config: &SdkConfig) -> Self {
+    /// To init the struct with main settings
+    pub async fn with_settings(settings: &impl SettingsProvider) -> Self {
+        let s3_config: AWSS3Config = settings.get_settings(S3_SETTINGS_NAME).unwrap();
+        let aws_config = aws_config::load_from_env().await;
+
         // Building AWS S3 config
-        let mut s3_config_builder = aws_sdk_s3::config::Builder::from(aws_config);
+        let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&aws_config);
 
         // this is necessary for it to work with localstack in test cases
         s3_config_builder.set_force_path_style(Some(true));

crates/orchestrator/src/database/mongodb/config.rs

@@ -1,7 +1,9 @@
+use serde::{Deserialize, Serialize};
 use utils::env_utils::get_env_var_or_panic;
 
 use crate::database::DatabaseConfig;
 
+#[derive(Debug, Serialize, Deserialize)]
 pub struct MongoDbConfig {
     pub url: String,
 }
@@ -11,3 +13,9 @@ impl DatabaseConfig for MongoDbConfig {
         Self { url: get_env_var_or_panic("MONGODB_CONNECTION_STRING") }
     }
 }
+
+impl Default for MongoDbConfig {
+    fn default() -> Self {
+        Self { url: get_env_var_or_panic("MONGODB_CONNECTION_STRING") }
+    }
+}

crates/orchestrator/src/database/mongodb/mod.rs

@@ -13,6 +13,7 @@ use mongodb::{
     options::{ClientOptions, ServerApi, ServerApiVersion},
     Client, Collection,
 };
+use utils::settings::SettingsProvider;
 use uuid::Uuid;
 
 use crate::database::mongodb::config::MongoDbConfig;
@@ -21,13 +22,17 @@ use crate::jobs::types::{JobItem, JobStatus, JobType};
 
 pub mod config;
 
+pub const MONGO_DB_SETTINGS: &str = "mongodb";
+
 pub struct MongoDb {
     client: Client,
 }
 
 impl MongoDb {
-    pub async fn new(config: MongoDbConfig) -> Self {
-        let mut client_options = ClientOptions::parse(config.url).await.expect("Failed to parse MongoDB Url");
+    pub async fn with_settings(settings: &impl SettingsProvider) -> Self {
+        let mongo_db_settings: MongoDbConfig = settings.get_settings(MONGO_DB_SETTINGS).unwrap();
+        let mut client_options =
+            ClientOptions::parse(mongo_db_settings.url).await.expect("Failed to parse MongoDB Url");
         // Set the server_api field of the client_options object to set the version of the Stable API on the
         // client
         let server_api = ServerApi::builder().version(ServerApiVersion::V1).build();
@@ -38,7 +43,7 @@ impl MongoDb {
         client.database("admin").run_command(doc! {"ping": 1}, None).await.expect("Failed to ping MongoDB deployment");
         println!("Pinged your deployment. You successfully connected to MongoDB!");
 
-        MongoDb { client }
+        Self { client }
     }
 
     /// Mongodb client uses Arc internally, reducing the cost of clone.