diff --git a/.env.example b/.env.example index 310ac77c..fc5dc314 100644 --- a/.env.example +++ b/.env.example @@ -1,50 +1,51 @@ +##### ORCHESTRATOR ##### + HOST= PORT= -DATABASE_URL= -MADARA_RPC_URL= -DA_LAYER= -SETTLEMENT_LAYER= -# Ethereum -ETHEREUM_PRIVATE_KEY= -ETHEREUM_RPC_URL= -MEMORY_PAGES_CONTRACT_ADDRESS= -STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS= +##### AWS CONFIG ##### +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_REGION= +AWS_ENDPOINT_URL= -# Starknet -STARKNET_PUBLIC_KEY= -STARNET_PRIVATE_KEY= -STARKNET_RPC_URL= -STARKNET_CAIRO_CORE_CONTRACT_ADDRESS= +##### STORAGE ##### -# MongoDB connection string -MONGODB_CONNECTION_STRING= +DATA_STORAGE= +AWS_S3_BUCKET_NAME= -# AWS -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= -AWS_DEFAULT_REGION= +##### QUEUE ##### -# SQS +QUEUE_PROVIDER= SQS_JOB_PROCESSING_QUEUE_URL= SQS_JOB_VERIFICATION_QUEUE_URL= SQS_JOB_HANDLE_FAILURE_QUEUE_URL= SQS_WORKER_TRIGGER_QUEUE_URL= -# S3 -AWS_S3_BUCKET_NAME= -AWS_S3_BUCKET_REGION= +##### DATABASE ##### +DATABASE= +MONGODB_CONNECTION_STRING= -# Ethereum Settlement -DEFAULT_SETTLEMENT_CLIENT_RPC= -DEFAULT_L1_CORE_CONTRACT_ADDRESS= +##### PROVER ##### -# Sharp Services +PROVER_SERVICE= SHARP_CUSTOMER_ID= +SHARP_URL= +# [IMP!!!] These are test certificates (they don't work) SHARP_USER_CRT= SHARP_USER_KEY= SHARP_SERVER_CRT= SHARP_PROOF_LAYOUT= +##### ON CHAIN CONFIG ##### + +DA_LAYER= +SETTLEMENT_LAYER= +ETHEREUM_RPC_URL= +MADARA_RPC_URL= +MEMORY_PAGES_CONTRACT_ADDRESS= +PRIVATE_KEY= +ETHEREUM_PRIVATE_KEY= +STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS= diff --git a/.env.test b/.env.test index 08b22c73..b87267ed 100644 --- a/.env.test +++ b/.env.test @@ -1,41 +1,36 @@ -##### AWS config ##### +##### ORCHESTRATOR ##### + +HOST=127.0.0.1 +PORT=3000 + +##### AWS CONFIG ##### AWS_ACCESS_KEY_ID="AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="AWS_SECRET_ACCESS_KEY" -AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket" -AWS_S3_BUCKET_REGION="us-east-1" +AWS_REGION="us-east-1" AWS_ENDPOINT_URL="http://localhost.localstack.cloud:4566" + +##### STORAGE ##### + +DATA_STORAGE="s3" +AWS_S3_BUCKET_NAME="madara-orchestrator-test-bucket" + +##### QUEUE ##### + +QUEUE_PROVIDER="sqs" SQS_JOB_PROCESSING_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_processing_queue" SQS_JOB_VERIFICATION_QUEUE_URL="http://sqs.us-east-1.localhost.localstack.cloud:4566/000000000000/madara_orchestrator_job_verification_queue" SQS_JOB_HANDLE_FAILURE_QUEUE_URL= SQS_WORKER_TRIGGER_QUEUE_URL= -AWS_DEFAULT_REGION="localhost" - -##### On chain config ##### - -MADARA_RPC_URL="http://localhost:3000" -ETHEREUM_RPC_URL="http://localhost:3001" -MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" -PRIVATE_KEY="0xdead" -# Private key of Test wallet provided by Anvil -ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" -STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead" -##### Config URLs ##### +##### DATABASE ##### -DA_LAYER="ethereum" -PROVER_SERVICE="sharp" -SETTLEMENT_LAYER="ethereum" -DATA_STORAGE="s3" +DATABASE="mongodb" MONGODB_CONNECTION_STRING="mongodb://localhost:27017" -DEFAULT_SETTLEMENT_CLIENT_RPC="http://localhost:3000" -# Ethereum Settlement -DEFAULT_L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4" -SHOULD_IMPERSONATE_ACCOUNT="true" -TEST_DUMMY_CONTRACT_ADDRESS="0xE5b6F5e695BA6E4aeD92B68c4CC8Df1160D69A81" +##### PROVER ##### -# Sharp Services +PROVER_SERVICE="sharp" SHARP_CUSTOMER_ID="sharp_consumer_id" SHARP_URL="http://127.0.0.1:5000" # [IMP!!!] These are test certificates (they don't work) @@ -43,3 +38,15 @@ SHARP_USER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUR4ekNDQXErZ0F3SUJBZ0lV SHARP_USER_KEY="LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRRFJ5bVQzWTJQdkFJL0gKVkwxaTBNU011ZDVQanhuRE9UbW5tODBscWd4aXhTVHpiM1Z0cUdkUVNiQTZJWVpueEIvc25RSVFLdjZPL09QWApKMStrOWt2VjVwSjZiN0o4SUVLeFBZTW95WHNOU3dCdHhwclVpRGRXeUNhSXNtOEFBOHVKVHNDNUZ1ZDdWcDhhClVaT1ZGWDAwbHltazBEY2hzSSt3cFN4NUdxOG1HbU8vV1BjaHRFYnhRSGFFZDdOS01wR3VnQ2xocm1mRGJvTWUKZXVxUHRtV0VNNjVzbmpFa2hXeHpMcG1VUzE2QjRwUEFDU25xN1o3QUEveXRVVUc2T0hKcDVhb1RFMmdTbXl3RgpGZVpzVktObWFNVHlGRHBnd1BlNjl0blVtdzNSVkhNZjNmRDZkTTFMSzZzRDZ3WHFwMGRjQ3ZEdU5TQm8rZHhiCnQxWlFzdjhkQWdNQkFBRUNnZ0VBQU9mcDFiT2xLOVFKeXVlUHhjeDIvTkNVcUMxTEJDL01FdkEyUzVKWGFWbkcKbGhLR0pFb1U0Q0RoVk83dUlLYVZLTFZvMjk4RHFHUnBLM1d0RVE1TE40bytXYTcveTA5c1drMlVzbWxrVWFOZwpSaGtVZEJSK2dLNXVsQ3FKRml2dUJoTEQvRWlnQ1VWUGZKS2JtNG96TnpYcjVSMU5ENlV1aWFtODdtenlFcTBLCmZsVXlhR0RZNGdIdFNBOVBENVBFYlUveFpKeitKaHk5T2l3aVRXV0MrSHoyb2c3UWRDRDE2RlhGcit2VHpQN0MKb2tFb0VDZFNPRWlMalVENjBhS2ZxRmFCVm5MTkVudC9QSytmY1RBM05mNGtSMnFDNk9ZWjVFb09zYm1ka29ZTgpyU3NJZW9XblMxOEhvekZud2w3Z05wTUtjNmRzQzRBTldOVDFsTkhCb1FLQmdRRHlaUDFJSlppZUh6NlExaUVTCm5zd2tnblZCQUQ0SlVLR1ZDMHA3dk4yclNDZXh4c05ZZXFPTEEyZGZCUGpOVjd3blFKcUgxT05XellOMUJVSUUKeThLTCtFZVl6Q3RZa21LL21wSGJIMzNjd2tJODBuMHJROU1BalZMTlJ2YVVEOWp1NFBsRzFqaEFZUVVyTkViZQpKRlVpSk83aDVQa1llZG50SitqSHFpQnRoUUtCZ1FEZGtPbndmL0szYk4xenR0bXZQd0VicjhkVWJjRVh5NDFOCkl5VWwrZW1WSlgzYktKM0duNDZnQ2RsTTdkYmpwS3JVZ3oxL2JsZTgvMkVFckJvSEFRNkMrU2pEaGhvL01CbnIKekZheTBoK3YxbjBnZnNNVzRoOEF4cEFwc25OYnh6K2g1Wm5uSnRTd0srUjB3U0VJVVEzRjAxL2hMWWhLQ2l5OApwbW5HQi9hU3VRS0JnRzdxd1cvVExGd214ZlYyMXBsenFzeUdHZXVObGRXalhOMGIxcEI2b3lDdW11TmhwYUFHCk5uSDFNOGNxT2tPVWd4ZWZHMWRPbGx6eEc5ZGZlWTlDUWhyVW1NYVZucndmK0NuZkxDRU43d1VtcXpLenl1MFMKVXlwc2dOaElRYXNNK1dLTjllTnhRVHBNYXhZVERONjMxM0VSWDNKazJZdFdydDh6cFBSQXFDZ1ZBb0dCQU54egpUa0NMbmJ6aFphbTNlZm9DenlCMEVma3dSdHBkSGxkc3E0NlFqTmRuK1VSd3NpTXBLR2lWeEE3bDZsU1B4NlV3CmU2VHA3Z1JQZUlHRWwxVDJ1VENacGZSODNtcVdlb1FCeVJXZE9nZmplcFkxYWZpL3ZhY3c2Y21ERTRKeXloNVUKYTMveFE5ZVJwSHFDbWxKREMxZ1V5eVlwL3B2a2FjUytNeW5sVEhHSkFvR0FQekdTSzdXOHBUYldSVEFoaTVrSQpwZk5kWk1tcnRodUxNT3F6TGhyRjZublpldk9OdTBoYXVhZktlVElFd2w0clhYZHFKQlJBaWZKMFFsLzZKWFFkCmd1VzFrZWk1Ui8rUFZ5eUhab042c3NXSTNWYklwUUloUmt6UENnTDZhbHEwSzFpT1dlV1lIOHdORGRRdlB1T2UKRkZPOEovSzNxV0NtWjU0ODBBbTNhT0U9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K" SHARP_SERVER_CRT="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURhekNDQWxPZ0F3SUJBZ0lVRUR0Rjd0YVNTUnVQQTJ6Uk1aNWNzY2JCRm5jd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNTVTR4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TkRBNE1UTXhNekEzTVROYUZ3MHlOVEE0Ck1UTXhNekEzTVROYU1FVXhDekFKQmdOVkJBWVRBa2xPTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0VpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRFRHcEEwNEZ1QlNFaE5PNVYvMGxTaDkvSEgxeVRZT2dRVFdoOG43eDlRCnZGMHpvZFZueVFIdjE5elU5eVdia2xvOEkvOXFBVm9lRzdXTnpUVFg2Q295ZlNjb1YvazN0Q2UwVnVWMlFJTVQKdW82SzJSU05CVHB1TlNqNTlzUiszVTQ2OFRBQnY0YVpsYjU4TU5CRXM3MVRieVpLRHBGRVRkMkg3T0ZKajg4QQpNRi9MaXJkeDZPOFdZL0tDeisxd1ZXL1JRdytYYjRJSWx4bXJFOC9UZ3FNSEo4dFUxYkZiOWJNcTEvOTN5YWtJClU1V2J2NVhXKzFwZFVyTUFNcTFFaC9vZThMN2pFaFdvZXZrNzgyU0kwUk0xeG5MaEtrUUVBYXd6Zkg2ODZiR2YKUHQ3RkFIQ1pGaWJ4KzZzSkg0R1M3S25iK0x5bk9ud3phMWZPUXZEZmcvRm5BZ01CQUFHalV6QlJNQjBHQTFVZApEZ1FXQkJUYlFUdmlUTW1xNXlNK2ZJRVI4VjdTZk1pK3B6QWZCZ05WSFNNRUdEQVdnQlRiUVR2aVRNbXE1eU0rCmZJRVI4VjdTZk1pK3B6QVBCZ05WSFJNQkFmOEVCVEFEQVFIL01BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRREYKTllyRnpBa2RIVkhjUkd5SUNsTi9IVGswaldOcTVSdTB1RUpDQ21Dbm9ZY1pRSTlDQlcwTkl3dGpZUkpTKzR1UwordWh4VWpSYTA5YXdOWDhvYmU0dDZjK25HRnhZMGZqamk0cGZnbU1kMWNJeGdsM3E3Nlp0ZkllRGR6alRLRXN1CjRFUTVadnEwMnJvTEZ0ZjEvL3dRVG0xNkNKdFpGWnhNZ1phYnNxc2JRc3M2dWdMUGtTTmdBWjI1L2VhcWhnQ20KTjFUV2FxL0xJMVBLSkxPK085NFlMa2FsNVpyOTJCOXk4Q0VKVUVuSTA1R1N1MmJUOFM2a0ZBMEpadEszTW9SbwpqRWZWV1lQVHR5TFR4amNvRndCcDlHaXZYSDdSdHBxMDlmSmFhU1pNekxmNGlyNHpBdXprbExBNWZvampPNXlKCllnYlVaQUU2aS81N1NFWjR3VmxTCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" SHARP_PROOF_LAYOUT="small" + +##### ON CHAIN CONFIG ##### + +DA_LAYER="ethereum" +SETTLEMENT_LAYER="ethereum" +SETTLEMENT_RPC_URL="http://localhost:3001" +MADARA_RPC_URL="http://localhost:3000" +L1_CORE_CONTRACT_ADDRESS="0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4" +MEMORY_PAGES_CONTRACT_ADDRESS="0x000000000000000000000000000000000001dead" +PRIVATE_KEY="0xdead" +ETHEREUM_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS="0x000000000000000000000000000000000002dead" diff --git a/CHANGELOG.md b/CHANGELOG.md index 60de9a90..952e7162 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## Changed +- refactor AWS config usage and clean .env files - GitHub's coverage CI yml file for localstack and db testing. - Orchestrator :Moved TestConfigBuilder to `config.rs` in tests folder. - `.env` file requires two more variables which are queue urls for processing diff --git a/crates/da-clients/ethereum/src/config.rs b/crates/da-clients/ethereum/src/config.rs index b50604b2..34613486 100644 --- a/crates/da-clients/ethereum/src/config.rs +++ b/crates/da-clients/ethereum/src/config.rs @@ -19,14 +19,14 @@ pub struct EthereumDaConfig { impl DaConfig for EthereumDaConfig { fn new_from_env() -> Self { Self { - rpc_url: get_env_var_or_panic("ETHEREUM_RPC_URL"), + rpc_url: get_env_var_or_panic("SETTLEMENT_RPC_URL"), memory_pages_contract: get_env_var_or_panic("MEMORY_PAGES_CONTRACT_ADDRESS"), private_key: get_env_var_or_panic("PRIVATE_KEY"), } } async fn build_client(&self) -> EthereumDaClient { let client = - RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse ETHEREUM_RPC_URL")); + RpcClient::new_http(Url::from_str(self.rpc_url.as_str()).expect("Failed to parse SETTLEMENT_RPC_URL")); let provider = ProviderBuilder::<_, Ethereum>::new().on_client(client); EthereumDaClient { provider } diff --git a/crates/orchestrator/src/config.rs b/crates/orchestrator/src/config.rs index 447b25ec..e3b7b975 100644 --- a/crates/orchestrator/src/config.rs +++ b/crates/orchestrator/src/config.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use crate::data_storage::aws_s3::config::{AWSS3Config, AWSS3ConfigType}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use arc_swap::{ArcSwap, Guard}; +use aws_config::SdkConfig; use da_client_interface::{DaClient, DaConfig}; use dotenvy::dotenv; use ethereum_da_client::config::EthereumDaConfig; @@ -54,10 +55,16 @@ pub async fn init_config() -> Config { )); // init database - let database = Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await); + let database = build_database_client().await; + + // init AWS + let aws_config = aws_config::load_from_env().await; // init the queue - let queue = Box::new(SqsQueue {}); + // TODO: we use omniqueue for now which doesn't support loading AWS config + // from `SdkConfig`. We can later move to using `aws_sdk_sqs`. This would require + // us stop using the generic omniqueue abstractions for message ack/nack + let queue = build_queue_client(&aws_config); let da_client = build_da_client().await; @@ -65,7 +72,7 @@ pub async fn init_config() -> Config { let settlement_client = build_settlement_client(&settings_provider).await; let prover_client = build_prover_service(&settings_provider); - let storage_client = build_storage_client().await; + let storage_client = build_storage_client(&aws_config).await; Config::new(Arc::new(provider), da_client, prover_client, settlement_client, database, queue, storage_client) } @@ -177,9 +184,23 @@ pub async fn build_settlement_client( } } -pub async fn build_storage_client() -> Box { +pub async fn build_storage_client(aws_config: &SdkConfig) -> Box { match get_env_var_or_panic("DATA_STORAGE").as_str() { - "s3" => Box::new(AWSS3::new(AWSS3ConfigType::WithoutEndpoint(AWSS3Config::new_from_env())).await), + "s3" => Box::new(AWSS3::new(AWSS3Config::new_from_env(), aws_config)), _ => panic!("Unsupported Storage Client"), } } + +pub fn build_queue_client(_aws_config: &SdkConfig) -> Box { + match get_env_var_or_panic("QUEUE_PROVIDER").as_str() { + "sqs" => Box::new(SqsQueue {}), + _ => panic!("Unsupported Queue Client"), + } +} + +pub async fn build_database_client() -> Box { + match get_env_var_or_panic("DATABASE").as_str() { + "mongodb" => Box::new(MongoDb::new(MongoDbConfig::new_from_env()).await), + _ => panic!("Unsupported Database Client"), + } +} diff --git a/crates/orchestrator/src/data_storage/aws_s3/config.rs b/crates/orchestrator/src/data_storage/aws_s3/config.rs index 0970d80d..665caeb4 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/config.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/config.rs @@ -2,64 +2,17 @@ use utils::env_utils::get_env_var_or_panic; use crate::data_storage::DataStorageConfig; -/// Represents the type of the config which one wants to pass to create the client -#[derive(Clone)] -pub enum AWSS3ConfigType { - WithEndpoint(S3LocalStackConfig), - WithoutEndpoint(AWSS3Config), -} - /// Represents AWS S3 config struct with all the necessary variables. #[derive(Clone)] pub struct AWSS3Config { - /// AWS ACCESS KEY ID - pub s3_key_id: String, - /// AWS ACCESS KEY SECRET - pub s3_key_secret: String, /// S3 Bucket Name - pub s3_bucket_name: String, - /// S3 Bucket region - pub s3_bucket_region: String, -} - -/// Represents AWS S3 config struct with all the necessary variables. -#[derive(Clone)] -pub struct S3LocalStackConfig { - /// AWS ACCESS KEY ID - pub s3_key_id: String, - /// AWS ACCESS KEY SECRET - pub s3_key_secret: String, - /// S3 Bucket Name - pub s3_bucket_name: String, - /// S3 Bucket region - pub s3_bucket_region: String, - /// Endpoint url - pub endpoint_url: String, + pub bucket_name: String, } /// Implementation of `DataStorageConfig` for `AWSS3Config` impl DataStorageConfig for AWSS3Config { /// To return the config struct by creating it from the environment variables. fn new_from_env() -> Self { - Self { - s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"), - s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"), - s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"), - s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"), - } - } -} - -/// Implementation of `DataStorageConfig` for `S3LocalStackConfig` -impl DataStorageConfig for S3LocalStackConfig { - /// To return the config struct by creating it from the environment variables. - fn new_from_env() -> Self { - Self { - s3_key_id: get_env_var_or_panic("AWS_ACCESS_KEY_ID"), - s3_key_secret: get_env_var_or_panic("AWS_SECRET_ACCESS_KEY"), - s3_bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME"), - s3_bucket_region: get_env_var_or_panic("AWS_S3_BUCKET_REGION"), - endpoint_url: get_env_var_or_panic("AWS_ENDPOINT_URL"), - } + Self { bucket_name: get_env_var_or_panic("AWS_S3_BUCKET_NAME") } } } diff --git a/crates/orchestrator/src/data_storage/aws_s3/mod.rs b/crates/orchestrator/src/data_storage/aws_s3/mod.rs index 990a5d0a..0f920c7a 100644 --- a/crates/orchestrator/src/data_storage/aws_s3/mod.rs +++ b/crates/orchestrator/src/data_storage/aws_s3/mod.rs @@ -1,7 +1,7 @@ -use crate::data_storage::aws_s3::config::AWSS3ConfigType; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::DataStorage; use async_trait::async_trait; -use aws_sdk_s3::config::{Builder, Credentials, Region}; +use aws_config::SdkConfig; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::Client; use bytes::Bytes; @@ -13,7 +13,7 @@ pub mod config; /// AWSS3 represents AWS S3 client object containing the client and the config itself. pub struct AWSS3 { client: Client, - config: AWSS3ConfigType, + bucket: String, } /// Implementation for AWS S3 client. Contains the function for : @@ -22,63 +22,17 @@ pub struct AWSS3 { impl AWSS3 { /// Initializes a new AWS S3 client by passing the config /// and returning it. - pub async fn new(config: AWSS3ConfigType) -> Self { - let (config_builder, config) = match config { - AWSS3ConfigType::WithoutEndpoint(config) => { - let (credentials, region) = get_credentials_and_region_from_config( - config.s3_key_id.clone(), - config.s3_key_secret.clone(), - config.s3_bucket_region.clone(), - ); - ( - Builder::new().region(region).credentials_provider(credentials).force_path_style(true), - AWSS3ConfigType::WithoutEndpoint(config), - ) - } - AWSS3ConfigType::WithEndpoint(config) => { - let (credentials, region) = get_credentials_and_region_from_config( - config.s3_key_id.clone(), - config.s3_key_secret.clone(), - config.s3_bucket_region.clone(), - ); - ( - Builder::new() - .region(region) - .credentials_provider(credentials) - .force_path_style(true) - .endpoint_url(config.endpoint_url.clone()), - AWSS3ConfigType::WithEndpoint(config), - ) - } - }; - - let conf = config_builder.build(); - + pub fn new(s3_config: AWSS3Config, aws_config: &SdkConfig) -> Self { // Building AWS S3 config - let client = Client::from_conf(conf); - - Self { client, config } - } + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(aws_config); - pub fn get_bucket_name(&self) -> String { - match self.config.clone() { - AWSS3ConfigType::WithEndpoint(config) => config.s3_bucket_name, - AWSS3ConfigType::WithoutEndpoint(config) => config.s3_bucket_name, - } + // this is necessary for it to work with localstack in test cases + s3_config_builder.set_force_path_style(Some(true)); + let client = Client::from_conf(s3_config_builder.build()); + Self { client, bucket: s3_config.bucket_name } } } -/// Return the constructed `Credentials` and `Region` -fn get_credentials_and_region_from_config( - s3_key_id: String, - s3_key_secret: String, - s3_bucket_region: String, -) -> (Credentials, Region) { - let credentials = Credentials::new(s3_key_id, s3_key_secret, None, None, "loaded_from_custom_env"); - let region = Region::new(s3_bucket_region); - (credentials, region) -} - /// Implementation of `DataStorage` for `AWSS3` /// contains the function for getting the data and putting the data /// by taking the key as an argument. @@ -86,7 +40,7 @@ fn get_credentials_and_region_from_config( impl DataStorage for AWSS3 { /// Function to get the data from S3 bucket by Key. async fn get_data(&self, key: &str) -> Result { - let response = self.client.get_object().bucket(self.get_bucket_name()).key(key).send().await?; + let response = self.client.get_object().bucket(&self.bucket).key(key).send().await?; let data_stream = response.body.collect().await.expect("Failed to convert body into AggregatedBytes."); let data_bytes = data_stream.into_bytes(); Ok(data_bytes) @@ -96,7 +50,7 @@ impl DataStorage for AWSS3 { async fn put_data(&self, data: Bytes, key: &str) -> Result<()> { self.client .put_object() - .bucket(self.get_bucket_name()) + .bucket(&self.bucket) .key(key) .body(ByteStream::from(data)) .content_type("application/json") diff --git a/crates/orchestrator/src/tests/common/mod.rs b/crates/orchestrator/src/tests/common/mod.rs index 9d56b7e3..ee29ac34 100644 --- a/crates/orchestrator/src/tests/common/mod.rs +++ b/crates/orchestrator/src/tests/common/mod.rs @@ -7,8 +7,9 @@ use aws_config::Region; use mongodb::Client; use rstest::*; use serde::Deserialize; +use utils::env_utils::get_env_var_or_panic; -use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; use crate::database::mongodb::config::MongoDbConfig; @@ -84,5 +85,7 @@ pub struct MessagePayloadType { } pub async fn get_storage_client() -> Box { - Box::new(AWSS3::new(AWSS3ConfigType::WithEndpoint(S3LocalStackConfig::new_from_env())).await) + let aws_config = + aws_config::load_from_env().await.into_builder().endpoint_url(get_env_var_or_panic("AWS_ENDPOINT_URL")).build(); + Box::new(AWSS3::new(AWSS3Config::new_from_env(), &aws_config)) } diff --git a/crates/orchestrator/src/tests/data_storage/mod.rs b/crates/orchestrator/src/tests/data_storage/mod.rs index d127917a..a3055acb 100644 --- a/crates/orchestrator/src/tests/data_storage/mod.rs +++ b/crates/orchestrator/src/tests/data_storage/mod.rs @@ -1,7 +1,6 @@ -use crate::data_storage::aws_s3::config::{AWSS3ConfigType, S3LocalStackConfig}; +use crate::data_storage::aws_s3::config::AWSS3Config; use crate::data_storage::aws_s3::AWSS3; use crate::data_storage::{DataStorage, DataStorageConfig}; -use crate::tests::config::TestConfigBuilder; use bytes::Bytes; use rstest::rstest; use serde_json::json; @@ -14,12 +13,12 @@ use utils::env_utils::get_env_var_or_panic; #[rstest] #[tokio::test] async fn test_put_and_get_data_s3() -> color_eyre::Result<()> { - TestConfigBuilder::new().build().await; - dotenvy::from_filename("../.env.test")?; - let config = S3LocalStackConfig::new_from_env(); - let s3_client = AWSS3::new(AWSS3ConfigType::WithEndpoint(config)).await; + let config = AWSS3Config::new_from_env(); + let aws_config = + aws_config::load_from_env().await.into_builder().endpoint_url(get_env_var_or_panic("AWS_ENDPOINT_URL")).build(); + let s3_client = AWSS3::new(config, &aws_config); s3_client.build_test_bucket(&get_env_var_or_panic("AWS_S3_BUCKET_NAME")).await.unwrap(); let mock_data = json!( diff --git a/crates/settlement-clients/ethereum/src/config.rs b/crates/settlement-clients/ethereum/src/config.rs index 294c2a54..3d038390 100644 --- a/crates/settlement-clients/ethereum/src/config.rs +++ b/crates/settlement-clients/ethereum/src/config.rs @@ -6,8 +6,8 @@ use url::Url; use utils::env_utils::get_env_var_or_panic; pub const ENV_CORE_CONTRACT_ADDRESS: &str = "STARKNET_SOLIDITY_CORE_CONTRACT_ADDRESS"; -pub const DEFAULT_SETTLEMENT_CLIENT_RPC: &str = "DEFAULT_SETTLEMENT_CLIENT_RPC"; -pub const DEFAULT_L1_CORE_CONTRACT_ADDRESS: &str = "DEFAULT_L1_CORE_CONTRACT_ADDRESS"; +pub const SETTLEMENT_RPC_URL: &str = "SETTLEMENT_RPC_URL"; +pub const L1_CORE_CONTRACT_ADDRESS: &str = "L1_CORE_CONTRACT_ADDRESS"; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EthereumSettlementConfig { @@ -17,9 +17,8 @@ pub struct EthereumSettlementConfig { impl SettlementConfig for EthereumSettlementConfig { fn new_from_env() -> Self { - let rpc_url = get_env_var_or_panic(DEFAULT_SETTLEMENT_CLIENT_RPC); - let rpc_url = - Url::from_str(&rpc_url).unwrap_or_else(|_| panic!("Failed to parse {}", DEFAULT_SETTLEMENT_CLIENT_RPC)); + let rpc_url = get_env_var_or_panic(SETTLEMENT_RPC_URL); + let rpc_url = Url::from_str(&rpc_url).unwrap_or_else(|_| panic!("Failed to parse {}", SETTLEMENT_RPC_URL)); let core_contract_address = get_env_var_or_panic(ENV_CORE_CONTRACT_ADDRESS); Self { rpc_url, core_contract_address } } @@ -28,8 +27,8 @@ impl SettlementConfig for EthereumSettlementConfig { impl Default for EthereumSettlementConfig { fn default() -> Self { Self { - rpc_url: get_env_var_or_panic(DEFAULT_SETTLEMENT_CLIENT_RPC).parse().unwrap(), - core_contract_address: get_env_var_or_panic(DEFAULT_L1_CORE_CONTRACT_ADDRESS), + rpc_url: get_env_var_or_panic(SETTLEMENT_RPC_URL).parse().unwrap(), + core_contract_address: get_env_var_or_panic(L1_CORE_CONTRACT_ADDRESS), } } }