From 2af5d8dad9885ead763c6e4f550ca6c73c24966f Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 15:05:21 +0100 Subject: [PATCH 01/77] feat: Add S3 support --- Cargo.toml | 3 + crates/rattler-bin/Cargo.toml | 2 +- crates/rattler-bin/src/commands/create.rs | 1 + crates/rattler_networking/Cargo.toml | 4 + crates/rattler_networking/src/lib.rs | 5 + .../rattler_networking/src/s3_middleware.rs | 91 +++++++++++++++++++ 6 files changed, 105 insertions(+), 1 deletion(-) create mode 100644 crates/rattler_networking/src/s3_middleware.rs diff --git a/Cargo.toml b/Cargo.toml index b49b93c9e..37cca7764 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,9 @@ getrandom = { version = "0.2.15", default-features = false } glob = "0.3.1" google-cloud-auth = { version = "0.17.2", default-features = false } google-cloud-token = "0.1.2" +aws-config = "1.5.12" +aws-sdk-s3 = "1.67.0" +aws-runtime = "1.5.2" hex = "0.4.3" hex-literal = "0.4.1" http = "1.2" diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 752b09c15..5cf9df0df 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -29,7 +29,7 @@ indicatif = { workspace = true } once_cell = { workspace = true } rattler = { path="../rattler", version = "0.28.9", default-features = false, features = ["indicatif"] } rattler_conda_types = { path="../rattler_conda_types", version = "0.29.7", default-features = false } -rattler_networking = { path="../rattler_networking", version = "0.21.9", default-features = false, features = ["gcs"] } +rattler_networking = { path="../rattler_networking", version = "0.21.9", default-features = false, features = ["gcs", "s3"] } rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.29", default-features = false, features = ["gateway"] } rattler_solve = { path="../rattler_solve", version = "1.3.1", default-features = false, features = ["resolvo", "libsolv_c"] } rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.15", default-features = false } diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 73bad476a..d7a093e76 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -153,6 +153,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { authentication_storage, ))) .with(rattler_networking::OciMiddleware) + .with(rattler_networking::S3Middleware::new(None, None, None).await) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 53447e68e..bede8b9d7 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -15,6 +15,7 @@ default = ["native-tls"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] +s3 = ["aws-config", "aws-sdk-s3", "aws-runtime"] [dependencies] anyhow = { workspace = true } @@ -25,6 +26,9 @@ dirs = { workspace = true } fslock = { workspace = true } google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-s3 = { workspace = true, optional = true } +aws-runtime = { workspace = true, optional = true } http = { workspace = true } itertools = { workspace = true } keyring = { workspace = true, features = ["apple-native", "windows-native", "async-secret-service", "async-io", "crypto-rust"] } diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index c8ba7ff18..23c4ffce1 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -11,6 +11,11 @@ pub mod gcs_middleware; #[cfg(feature = "gcs")] pub use gcs_middleware::GCSMiddleware; +#[cfg(feature = "s3")] +pub mod s3_middleware; +#[cfg(feature = "s3")] +pub use s3_middleware::S3Middleware; + pub mod authentication_middleware; pub mod authentication_storage; diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs new file mode 100644 index 000000000..ac27dcd53 --- /dev/null +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -0,0 +1,91 @@ +//! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use url::Url; + +/// S3 middleware to authenticate requests +pub struct S3Middleware { + client: aws_sdk_s3::Client +} + +impl S3Middleware { + /// Create a new S3 middleware + pub async fn new(config_file: Option<&str>, profile: Option<&str>, force_path_style: Option) -> Self { + let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); + if let Some(config_file) = config_file { + builder = builder.with_file(aws_runtime::env_config::file::EnvConfigFileKind::Config, config_file) + } + let env_config_files = builder.build(); + + let mut builder = aws_config::defaults(BehaviorVersion::latest()); + if config_file.is_some() { + builder = builder.profile_files(env_config_files) + }; + if let Some(profile) = profile { + builder = builder.profile_name(profile) + }; + let sdk_config = builder.load().await; + + let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); + if let Some(force_path_style) = force_path_style { + builder = builder.force_path_style(force_path_style) + }; + let s3_config = builder.build(); + + let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); + Self { + client + } + } + + /// Generate a presigned S3 GetObject request + async fn generate_presigned_s3_request(&self, bucket_name: &str, key: &str) -> MiddlewareResult { + let builder = self.client.get_object().bucket(bucket_name).key(key); + Ok(builder.presigned(PresigningConfig::expires_in(std::time::Duration::from_secs(300)).unwrap()).await.unwrap()) + } +} + + +#[async_trait] +impl Middleware for S3Middleware { + /// Create a new authentication middleware for S3 + async fn handle( + &self, + mut req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + if req.url().scheme() == "s3" { + let url = req.url().clone(); + let bucket_name = url.host_str().expect("Host should be present in S3 URL"); + let key = url.path(); + let presigned_request = self.generate_presigned_s3_request(bucket_name, key).await?; + + *req.url_mut() = Url::parse(presigned_request.uri()).unwrap(); + } + next.run(req, extensions).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_presigned_s3_request() { + if std::env::var("AWS_ACCESS_KEY_ID").is_err() { + eprintln!("Skipping test as AWS_ACCESS_KEY_ID is not set"); + return; + }; + eprintln!("Running test"); + + let middleware = S3Middleware::new(None, None, None).await; + + let presigned = middleware.generate_presigned_s3_request("rattler-s3-testing", "input.txt").await.unwrap(); + assert!(presigned.uri().to_string().starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/input.txt?"), "Unexpected presigned URL: {:?}", presigned.uri()); + panic!("{:?}", presigned.uri()); + } +} From 72d763affe6b3213a50dbbfd060d065bfadc7285 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 15:08:24 +0100 Subject: [PATCH 02/77] fmt --- .../rattler_networking/src/s3_middleware.rs | 46 +++++++++++++------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index ac27dcd53..2a2687a50 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -8,15 +8,22 @@ use url::Url; /// S3 middleware to authenticate requests pub struct S3Middleware { - client: aws_sdk_s3::Client + client: aws_sdk_s3::Client, } impl S3Middleware { /// Create a new S3 middleware - pub async fn new(config_file: Option<&str>, profile: Option<&str>, force_path_style: Option) -> Self { + pub async fn new( + config_file: Option<&str>, + profile: Option<&str>, + force_path_style: Option, + ) -> Self { let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); if let Some(config_file) = config_file { - builder = builder.with_file(aws_runtime::env_config::file::EnvConfigFileKind::Config, config_file) + builder = builder.with_file( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + config_file, + ) } let env_config_files = builder.build(); @@ -28,7 +35,7 @@ impl S3Middleware { builder = builder.profile_name(profile) }; let sdk_config = builder.load().await; - + let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); if let Some(force_path_style) = force_path_style { builder = builder.force_path_style(force_path_style) @@ -36,19 +43,23 @@ impl S3Middleware { let s3_config = builder.build(); let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); - Self { - client - } + Self { client } } /// Generate a presigned S3 GetObject request - async fn generate_presigned_s3_request(&self, bucket_name: &str, key: &str) -> MiddlewareResult { + async fn generate_presigned_s3_request( + &self, + bucket_name: &str, + key: &str, + ) -> MiddlewareResult { let builder = self.client.get_object().bucket(bucket_name).key(key); - Ok(builder.presigned(PresigningConfig::expires_in(std::time::Duration::from_secs(300)).unwrap()).await.unwrap()) + Ok(builder + .presigned(PresigningConfig::expires_in(std::time::Duration::from_secs(300)).unwrap()) + .await + .unwrap()) } } - #[async_trait] impl Middleware for S3Middleware { /// Create a new authentication middleware for S3 @@ -84,8 +95,17 @@ mod tests { let middleware = S3Middleware::new(None, None, None).await; - let presigned = middleware.generate_presigned_s3_request("rattler-s3-testing", "input.txt").await.unwrap(); - assert!(presigned.uri().to_string().starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/input.txt?"), "Unexpected presigned URL: {:?}", presigned.uri()); - panic!("{:?}", presigned.uri()); + let presigned = middleware + .generate_presigned_s3_request("rattler-s3-testing", "input.txt") + .await + .unwrap(); + assert!( + presigned + .uri() + .to_string() + .starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/input.txt?"), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); } } From 973b183db7095bf4326665d4f308213aeaf272f2 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Thu, 2 Jan 2025 17:26:18 +0100 Subject: [PATCH 03/77] Add unit tests + empty integration test --- Cargo.toml | 1 + crates/rattler_networking/Cargo.toml | 13 +- .../rattler_networking/src/s3_middleware.rs | 224 ++++++++++++++++-- .../tests/integration_test.rs | 92 +++++++ pixi.toml | 6 +- 5 files changed, 312 insertions(+), 24 deletions(-) create mode 100644 crates/rattler_networking/tests/integration_test.rs diff --git a/Cargo.toml b/Cargo.toml index 37cca7764..468a93b4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,6 +129,7 @@ serde-value = "0.7.0" serde_with = "3.11.0" serde_yaml = "0.9.34" serde-untagged = "0.1.6" +serial_test = "0.4.0" sha2 = "0.10.8" shlex = "1.3.0" similar-asserts = "1.6.0" diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index bede8b9d7..1798ccf3b 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls"] +default = ["native-tls", "s3"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] @@ -31,7 +31,13 @@ aws-sdk-s3 = { workspace = true, optional = true } aws-runtime = { workspace = true, optional = true } http = { workspace = true } itertools = { workspace = true } -keyring = { workspace = true, features = ["apple-native", "windows-native", "async-secret-service", "async-io", "crypto-rust"] } +keyring = { workspace = true, features = [ + "apple-native", + "windows-native", + "async-secret-service", + "async-io", + "crypto-rust", +] } netrc-rs = { workspace = true } reqwest = { workspace = true, features = ["json"] } reqwest-middleware = { workspace = true } @@ -54,3 +60,6 @@ axum = { workspace = true } reqwest-retry = { workspace = true } sha2 = { workspace = true } temp-env = { workspace = true } +rstest = { workspace = true } +rand = { workspace = true } +serial_test = { workspace = true } diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 2a2687a50..868fc2f1a 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -18,23 +18,21 @@ impl S3Middleware { profile: Option<&str>, force_path_style: Option, ) -> Self { - let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); + let mut aws_config_builder = aws_config::defaults(BehaviorVersion::latest()); if let Some(config_file) = config_file { + let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); builder = builder.with_file( aws_runtime::env_config::file::EnvConfigFileKind::Config, config_file, - ) + ); + let env_config_files = builder.build(); + aws_config_builder = aws_config_builder.profile_files(env_config_files); } - let env_config_files = builder.build(); - let mut builder = aws_config::defaults(BehaviorVersion::latest()); - if config_file.is_some() { - builder = builder.profile_files(env_config_files) - }; if let Some(profile) = profile { - builder = builder.profile_name(profile) + aws_config_builder = aws_config_builder.profile_name(profile) }; - let sdk_config = builder.load().await; + let sdk_config = aws_config_builder.load().await; let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); if let Some(force_path_style) = force_path_style { @@ -47,7 +45,7 @@ impl S3Middleware { } /// Generate a presigned S3 GetObject request - async fn generate_presigned_s3_request( + pub async fn generate_presigned_s3_request( &self, bucket_name: &str, key: &str, @@ -83,29 +81,213 @@ impl Middleware for S3Middleware { #[cfg(test)] mod tests { + use std::collections::HashMap; + use super::*; + use rstest::{fixture, rstest}; + use serial_test::serial; + use tempfile::{tempdir, TempDir}; + + async fn with_env( + env: HashMap<&str, &str>, + f: impl FnOnce() -> std::pin::Pin + Send>>, + ) { + for (key, value) in &env { + std::env::set_var(key, value); + } + f().await; + for (key, _) in env { + std::env::remove_var(key); + } + } #[tokio::test] + #[serial] async fn test_presigned_s3_request() { - if std::env::var("AWS_ACCESS_KEY_ID").is_err() { - eprintln!("Skipping test as AWS_ACCESS_KEY_ID is not set"); - return; - }; - eprintln!("Running test"); + with_env( + HashMap::from([ + ("AWS_ACCESS_KEY_ID", "minioadmin"), + ("AWS_SECRET_ACCESS_KEY", "minioadmin"), + ("AWS_REGION", "eu-central-1"), + ("AWS_ENDPOINT_URL", "http://localhost:9000"), + ]), + move || { + Box::pin(async { + let middleware = S3Middleware::new(None, None, Some(true)).await; + + let presigned = middleware + .generate_presigned_s3_request( + "rattler-s3-testing", + "my-channel/noarch/repodata.json", + ) + .await + .unwrap(); + assert!( + presigned.uri().to_string().starts_with( + "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?" + ), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); + }) + }, + ) + .await; + } + + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_aws() { + with_env( + HashMap::from([ + ("AWS_ACCESS_KEY_ID", "minioadmin"), + ("AWS_SECRET_ACCESS_KEY", "minioadmin"), + ("AWS_REGION", "eu-central-1"), + ]), + move || { + Box::pin(async { + let middleware = S3Middleware::new(None, None, None).await; + + let presigned = middleware + .generate_presigned_s3_request( + "rattler-s3-testing", + "my-channel/noarch/repodata.json", + ) + .await + .unwrap(); + assert!( + presigned.uri().to_string().starts_with( + "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" + ), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); + }) + }, + ) + .await; + } + + #[fixture] + fn aws_config() -> (TempDir, std::path::PathBuf) { + let temp_dir = tempdir().unwrap(); + let aws_config = r#" +[profile default] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +region = eu-central-1 + +[profile packages] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +endpoint_url = http://localhost:8000 +region = eu-central-1 +"#; + let aws_config_path = temp_dir.path().join("aws.config"); + std::fs::write(&aws_config_path, aws_config).unwrap(); + (temp_dir, aws_config_path) + } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_custom_config(aws_config: (TempDir, std::path::PathBuf)) { + let middleware = S3Middleware::new(Some(aws_config.1.to_str().unwrap()), None, None).await; - let middleware = S3Middleware::new(None, None, None).await; + let presigned = middleware + .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") + .await + .unwrap(); + assert!( + presigned.uri().to_string().starts_with( + "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" + ), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); + } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_different_profile( + aws_config: (TempDir, std::path::PathBuf), + ) { + let middleware = + S3Middleware::new(Some(aws_config.1.to_str().unwrap()), Some("packages"), None).await; let presigned = middleware - .generate_presigned_s3_request("rattler-s3-testing", "input.txt") + .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") .await .unwrap(); assert!( - presigned - .uri() - .to_string() - .starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/input.txt?"), + presigned.uri().to_string().contains("localhost:8000"), "Unexpected presigned URL: {:?}", presigned.uri() ); } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_custom_config_from_env( + aws_config: (TempDir, std::path::PathBuf), + ) { + with_env( + HashMap::from([ + ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), + ("AWS_PROFILE", "packages"), + ]), + move || { + Box::pin(async move { + let middleware = S3Middleware::new(None, None, None).await; + + let presigned = middleware + .generate_presigned_s3_request( + "rattler-s3-testing", + "my-channel/noarch/repodata.json", + ) + .await + .unwrap(); + assert!( + presigned.uri().to_string().contains("localhost:8000"), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); + }) + }, + ) + .await; + } + + /// Test that environment variables take precedence over the configuration file. + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { + with_env( + HashMap::from([("AWS_ENDPOINT_URL", "http://localhost:9000")]), + move || { + let aws_config_path = aws_config.1.to_str().unwrap().to_string(); + Box::pin(async move { + let middleware = + S3Middleware::new(Some(&aws_config_path), "default".into(), None).await; + + let presigned = middleware + .generate_presigned_s3_request( + "rattler-s3-testing", + "my-channel/noarch/repodata.json", + ) + .await + .unwrap(); + assert!( + presigned.uri().to_string().contains("localhost:9000"), + "Unexpected presigned URL: {:?}", + presigned.uri() + ); + }) + }, + ) + .await; + } } diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs new file mode 100644 index 000000000..a6b3c81f3 --- /dev/null +++ b/crates/rattler_networking/tests/integration_test.rs @@ -0,0 +1,92 @@ +use std::collections::HashMap; + +use rattler_networking::S3Middleware; +use rstest::*; +use serial_test::serial; + +struct MinioServer { + handle: std::process::Child, + directory: tempfile::TempDir, +} + +impl MinioServer { + fn new() -> Self { + let directory = tempfile::tempdir().expect("Failed to create temp directory"); + let args = [ + "server", + directory.path().to_str().unwrap(), + "--address", + "127.0.0.1:9000", + ]; + let handle = std::process::Command::new("minio") + .args(&args) + .spawn() + .expect("Failed to start Minio server"); + eprintln!( + "Starting Minio server with args (PID={}): {:?}", + handle.id(), + args + ); + MinioServer { handle, directory } + } +} + +fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::process::Output { + let mut command = std::process::Command::new(cmd); + command.args(args); + for (key, value) in env { + command.env(key, value); + } + let output = command.output().expect("Failed to run command"); + if !output.status.success() { + eprintln!("Command failed: {:?}", output); + } + output +} + +fn init_channel() { + let env = &HashMap::from([( + "MC_HOST_local", + "http://minioadmin:minioadmin@localhost:9000", + )]); + run_subprocess("mc", &["mb", "local/rattler-s3-testing"], env); + run_subprocess( + "mc", + &[ + "cp", + "../../test-data/test-server/repo/noarch/repodata.json", + "local/rattler-s3-testing/my-channel/noarch/repodata.json", + ], + env, + ); + run_subprocess( + "mc", + &[ + "cp", + "../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2", + "local/rattler-s3-testing/my-channel/noarch/test-package-0.1-0.tar.bz2", + ], + env, + ); +} + +#[fixture] +fn minio_server() -> MinioServer { + let server = MinioServer::new(); + init_channel(); + server +} + +#[rstest] +#[tokio::test] +#[serial] +async fn test_presigned_s3_request() { + std::env::set_var("AWS_ACCESS_KEY_ID", "minioadmin"); + std::env::set_var("AWS_SECRET_ACCESS_KEY", "minioadmin"); + std::env::set_var("AWS_REGION", "eu-central-1"); + std::env::set_var("AWS_ENDPOINT_URL", "http://localhost:9000"); + + let middleware = S3Middleware::new(None, None, Some(true)).await; + + // TODO: Do install or search +} diff --git a/pixi.toml b/pixi.toml index 33691d02b..68c9e7a0f 100644 --- a/pixi.toml +++ b/pixi.toml @@ -26,6 +26,8 @@ make = "~=4.3" pkg-config = "~=0.29.2" rust = "~=1.80.0" cmake = "~=3.26.4" +minio-server = "*" +# minio-client = "*" # xref: https://github.com/conda-forge/staged-recipes/pull/28736 [target.linux-64.dependencies] clang = ">=18.1.8,<19.0" @@ -45,4 +47,6 @@ pre-commit-install = "pre-commit install" pre-commit-run = "pre-commit run" [environments] -lint = { features = ["lint"], no-default-feature = true, solve-group = "default" } +lint = { features = [ + "lint", +], no-default-feature = true, solve-group = "default" } From 58c2f107a2241d1f257f0dc887593150b375c4a9 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Thu, 2 Jan 2025 17:54:11 +0100 Subject: [PATCH 04/77] Implement integration test --- .../rattler_networking/src/s3_middleware.rs | 13 ++-- .../tests/integration_test.rs | 71 ++++++++++++++++--- 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 868fc2f1a..82aa7c04c 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -1,4 +1,6 @@ //! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket +use std::path::PathBuf; + use async_trait::async_trait; use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; @@ -14,7 +16,7 @@ pub struct S3Middleware { impl S3Middleware { /// Create a new S3 middleware pub async fn new( - config_file: Option<&str>, + config_file: Option<&PathBuf>, profile: Option<&str>, force_path_style: Option, ) -> Self { @@ -45,7 +47,7 @@ impl S3Middleware { } /// Generate a presigned S3 GetObject request - pub async fn generate_presigned_s3_request( + async fn generate_presigned_s3_request( &self, bucket_name: &str, key: &str, @@ -192,7 +194,7 @@ region = eu-central-1 #[tokio::test] #[serial] async fn test_presigned_s3_request_custom_config(aws_config: (TempDir, std::path::PathBuf)) { - let middleware = S3Middleware::new(Some(aws_config.1.to_str().unwrap()), None, None).await; + let middleware = S3Middleware::new(Some(&aws_config.1), None, None).await; let presigned = middleware .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") @@ -213,8 +215,7 @@ region = eu-central-1 async fn test_presigned_s3_request_different_profile( aws_config: (TempDir, std::path::PathBuf), ) { - let middleware = - S3Middleware::new(Some(aws_config.1.to_str().unwrap()), Some("packages"), None).await; + let middleware = S3Middleware::new(Some(&aws_config.1), Some("packages"), None).await; let presigned = middleware .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") @@ -268,7 +269,7 @@ region = eu-central-1 with_env( HashMap::from([("AWS_ENDPOINT_URL", "http://localhost:9000")]), move || { - let aws_config_path = aws_config.1.to_str().unwrap().to_string(); + let aws_config_path = aws_config.1; Box::pin(async move { let middleware = S3Middleware::new(Some(&aws_config_path), "default".into(), None).await; diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index a6b3c81f3..07340d35c 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,8 +1,12 @@ -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf, sync::Arc}; -use rattler_networking::S3Middleware; +use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage, S3Middleware}; +use reqwest::Client; use rstest::*; use serial_test::serial; +use tempfile::{tempdir, TempDir}; + +/* ----------------------------------- MINIO UTILS ---------------------------------- */ struct MinioServer { handle: std::process::Child, @@ -31,6 +35,13 @@ impl MinioServer { } } +impl Drop for MinioServer { + fn drop(&mut self) { + eprintln!("Shutting down Minio server (PID={})", self.handle.id()); + self.handle.kill().expect("Failed to kill Minio server"); + } +} + fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::process::Output { let mut command = std::process::Command::new(cmd); command.args(args); @@ -54,7 +65,9 @@ fn init_channel() { "mc", &[ "cp", - "../../test-data/test-server/repo/noarch/repodata.json", + PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") + .to_str() + .unwrap(), "local/rattler-s3-testing/my-channel/noarch/repodata.json", ], env, @@ -63,7 +76,9 @@ fn init_channel() { "mc", &[ "cp", - "../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2", + PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") + .to_str() + .unwrap(), "local/rattler-s3-testing/my-channel/noarch/test-package-0.1-0.tar.bz2", ], env, @@ -77,16 +92,50 @@ fn minio_server() -> MinioServer { server } +#[fixture] +fn aws_config() -> (TempDir, std::path::PathBuf) { + let temp_dir = tempdir().unwrap(); + let aws_config = r#" +[profile default] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +endpoint_url = http://localhost:9000 +region = eu-central-1 +"#; + let aws_config_path = temp_dir.path().join("aws.config"); + std::fs::write(&aws_config_path, aws_config).unwrap(); + (temp_dir, aws_config_path) +} + #[rstest] #[tokio::test] #[serial] -async fn test_presigned_s3_request() { - std::env::set_var("AWS_ACCESS_KEY_ID", "minioadmin"); - std::env::set_var("AWS_SECRET_ACCESS_KEY", "minioadmin"); - std::env::set_var("AWS_REGION", "eu-central-1"); - std::env::set_var("AWS_ENDPOINT_URL", "http://localhost:9000"); +async fn test_minio_download_repodata( + minio_server: MinioServer, + aws_config: (TempDir, std::path::PathBuf), +) { + let middleware = S3Middleware::new(Some(&aws_config.1), "default".into(), Some(true)).await; + + let download_client = Client::builder() + .no_gzip() + .build() + .expect("failed to create client"); + + let authentication_storage = AuthenticationStorage::default(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new( + authentication_storage, + ))) + .with(middleware) + .build(); - let middleware = S3Middleware::new(None, None, Some(true)).await; + let result = download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); - // TODO: Do install or search + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); } From aa3c9213a7466d4fb595993ef5fb9137ee50a4a7 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Thu, 2 Jan 2025 17:56:47 +0100 Subject: [PATCH 05/77] Add s3 to default features --- .github/workflows/rust-compile.yml | 94 +++++++++++++++++++--------- crates/rattler_networking/Cargo.toml | 2 +- 2 files changed, 66 insertions(+), 30 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index e60144ae2..6a1674853 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -1,6 +1,6 @@ on: push: - branches: [ main ] + branches: [main] pull_request: paths: # When we change pyproject.toml, we want to ensure that the maturin builds still work @@ -22,7 +22,7 @@ env: RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always - DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c + DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c,s3 jobs: check-rustdoc-links: @@ -54,31 +54,70 @@ jobs: build: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} - needs: [ format_and_lint ] + needs: [format_and_lint] strategy: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-22.04 } - - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } - - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } + - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-22.04 } + - { + name: "Linux-aarch64", + target: aarch64-unknown-linux-musl, + os: ubuntu-latest, + skip-tests: true, + } + - { + name: "Linux-arm", + target: arm-unknown-linux-musleabi, + os: ubuntu-latest, + use-cross: true, + skip-tests: true, + } # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } -# - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } - - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } + # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + - { + name: "Linux-powerpc64", + target: powerpc64-unknown-linux-gnu, + os: ubuntu-latest, + use-cross: true, + skip-tests: true, + } + - { + name: "Linux-powerpc64le", + target: powerpc64le-unknown-linux-gnu, + os: ubuntu-latest, + use-cross: true, + skip-tests: true, + } + + - { + name: "Linux-s390x", + target: s390x-unknown-linux-gnu, + os: ubuntu-latest, + use-cross: true, + skip-tests: true, + } + + - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } + - { + name: "macOS-aarch64", + target: aarch64-apple-darwin, + os: macOS-latest, + skip-tests: true, + } + + - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } + - { + name: "Windows-aarch64", + target: aarch64-pc-windows-msvc, + os: windows-latest, + skip-tests: true, + } steps: - name: Checkout source code uses: actions/checkout@v4 @@ -115,17 +154,17 @@ jobs: - name: Use rustls on musl targets. id: build-options - if: contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc') || startsWith(matrix.target, 's390x') + if: + contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc') || + startsWith(matrix.target, 's390x') run: | echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT - name: Build run: > - cargo build - --all-targets - --features ${{ env.DEFAULT_FEATURES }} - --target ${{ matrix.target }} - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} + cargo build --all-targets + --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} ${{ + steps.build-options.outputs.CARGO_BUILD_OPTIONS}} - name: Disable testing the tools crate if cross compiling id: test-options @@ -144,12 +183,9 @@ jobs: env: GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }} run: > - cargo nextest run - --workspace - --features ${{ env.DEFAULT_FEATURES }} - --target ${{ matrix.target }} - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} - ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} + cargo nextest run --workspace --features ${{ env.DEFAULT_FEATURES }} --target ${{ + matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} ${{ + steps.test-options.outputs.CARGO_TEST_OPTIONS}} - name: Run doctests if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 1798ccf3b..5ee094053 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls", "s3"] +default = ["native-tls"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] From 7820c5cb37874e0d111b3bef06ef713d7bfc9182 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Thu, 2 Jan 2025 18:05:39 +0100 Subject: [PATCH 06/77] Fix unwraps --- .../rattler_networking/src/s3_middleware.rs | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 82aa7c04c..906990f75 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -8,13 +8,14 @@ use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; use url::Url; -/// S3 middleware to authenticate requests +/// S3 middleware to authenticate requests. pub struct S3Middleware { client: aws_sdk_s3::Client, + expiration: std::time::Duration, } impl S3Middleware { - /// Create a new S3 middleware + /// Create a new S3 middleware. pub async fn new( config_file: Option<&PathBuf>, profile: Option<&str>, @@ -32,37 +33,43 @@ impl S3Middleware { } if let Some(profile) = profile { - aws_config_builder = aws_config_builder.profile_name(profile) + aws_config_builder = aws_config_builder.profile_name(profile); }; let sdk_config = aws_config_builder.load().await; let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); if let Some(force_path_style) = force_path_style { - builder = builder.force_path_style(force_path_style) + builder = builder.force_path_style(force_path_style); }; let s3_config = builder.build(); let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); - Self { client } + Self { + client, + expiration: std::time::Duration::from_secs(300), + } } - /// Generate a presigned S3 GetObject request + /// Generate a presigned S3 `GetObject` request. async fn generate_presigned_s3_request( &self, bucket_name: &str, key: &str, ) -> MiddlewareResult { let builder = self.client.get_object().bucket(bucket_name).key(key); - Ok(builder - .presigned(PresigningConfig::expires_in(std::time::Duration::from_secs(300)).unwrap()) + builder + .presigned( + PresigningConfig::expires_in(self.expiration) + .map_err(reqwest_middleware::Error::middleware)?, + ) .await - .unwrap()) + .map_err(reqwest_middleware::Error::middleware) } } #[async_trait] impl Middleware for S3Middleware { - /// Create a new authentication middleware for S3 + /// Create a new authentication middleware for S3. async fn handle( &self, mut req: Request, @@ -75,7 +82,8 @@ impl Middleware for S3Middleware { let key = url.path(); let presigned_request = self.generate_presigned_s3_request(bucket_name, key).await?; - *req.url_mut() = Url::parse(presigned_request.uri()).unwrap(); + *req.url_mut() = Url::parse(presigned_request.uri()) + .map_err(reqwest_middleware::Error::middleware)?; } next.run(req, extensions).await } From 1bd0889ee15230c5bdf019ab66e346cd70796a77 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Thu, 2 Jan 2025 18:44:09 +0100 Subject: [PATCH 07/77] Add middleware --- .../rattler_networking/src/s3_middleware.rs | 88 +- .../tests/integration_test.rs | 2 +- py-rattler/Cargo.lock | 875 +++++++++++++++++- py-rattler/Cargo.toml | 1 + py-rattler/rattler/networking/__init__.py | 16 +- py-rattler/rattler/networking/client.py | 13 +- py-rattler/rattler/networking/middleware.py | 33 +- py-rattler/src/lib.rs | 2 + py-rattler/src/networking/client.rs | 3 +- py-rattler/src/networking/middleware.rs | 39 +- 10 files changed, 983 insertions(+), 89 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 906990f75..36035b958 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -10,42 +10,54 @@ use url::Url; /// S3 middleware to authenticate requests. pub struct S3Middleware { - client: aws_sdk_s3::Client, + config_file: Option, + profile: Option, + force_path_style: Option, expiration: std::time::Duration, } +async fn create_client( + config_file: Option, + profile: Option, + force_path_style: Option, +) -> aws_sdk_s3::Client { + let mut aws_config_builder = aws_config::defaults(BehaviorVersion::latest()); + if let Some(config_file) = config_file { + let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); + builder = builder.with_file( + aws_runtime::env_config::file::EnvConfigFileKind::Config, + config_file, + ); + let env_config_files = builder.build(); + aws_config_builder = aws_config_builder.profile_files(env_config_files); + } + + if let Some(profile) = profile { + aws_config_builder = aws_config_builder.profile_name(profile); + }; + let sdk_config = aws_config_builder.load().await; + + let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); + if let Some(force_path_style) = force_path_style { + builder = builder.force_path_style(force_path_style); + }; + let s3_config = builder.build(); + + let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); + client +} + impl S3Middleware { /// Create a new S3 middleware. - pub async fn new( - config_file: Option<&PathBuf>, - profile: Option<&str>, + pub fn new( + config_file: Option, + profile: Option, force_path_style: Option, ) -> Self { - let mut aws_config_builder = aws_config::defaults(BehaviorVersion::latest()); - if let Some(config_file) = config_file { - let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); - builder = builder.with_file( - aws_runtime::env_config::file::EnvConfigFileKind::Config, - config_file, - ); - let env_config_files = builder.build(); - aws_config_builder = aws_config_builder.profile_files(env_config_files); - } - - if let Some(profile) = profile { - aws_config_builder = aws_config_builder.profile_name(profile); - }; - let sdk_config = aws_config_builder.load().await; - - let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); - if let Some(force_path_style) = force_path_style { - builder = builder.force_path_style(force_path_style); - }; - let s3_config = builder.build(); - - let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); Self { - client, + config_file, + profile, + force_path_style, expiration: std::time::Duration::from_secs(300), } } @@ -56,7 +68,13 @@ impl S3Middleware { bucket_name: &str, key: &str, ) -> MiddlewareResult { - let builder = self.client.get_object().bucket(bucket_name).key(key); + let client = create_client( + self.config_file.clone(), + self.profile.clone(), + self.force_path_style, + ) + .await; + let builder = client.get_object().bucket(bucket_name).key(key); builder .presigned( PresigningConfig::expires_in(self.expiration) @@ -123,7 +141,7 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None, None, Some(true)).await; + let middleware = S3Middleware::new(None, None, Some(true)); let presigned = middleware .generate_presigned_s3_request( @@ -156,7 +174,7 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None, None, None).await; + let middleware = S3Middleware::new(None, None, None); let presigned = middleware .generate_presigned_s3_request( @@ -202,7 +220,7 @@ region = eu-central-1 #[tokio::test] #[serial] async fn test_presigned_s3_request_custom_config(aws_config: (TempDir, std::path::PathBuf)) { - let middleware = S3Middleware::new(Some(&aws_config.1), None, None).await; + let middleware = S3Middleware::new(Some(aws_config.1), None, None); let presigned = middleware .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") @@ -223,7 +241,7 @@ region = eu-central-1 async fn test_presigned_s3_request_different_profile( aws_config: (TempDir, std::path::PathBuf), ) { - let middleware = S3Middleware::new(Some(&aws_config.1), Some("packages"), None).await; + let middleware = S3Middleware::new(Some(aws_config.1), Some("packages".into()), None); let presigned = middleware .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") @@ -249,7 +267,7 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(None, None, None).await; + let middleware = S3Middleware::new(None, None, None); let presigned = middleware .generate_presigned_s3_request( @@ -280,7 +298,7 @@ region = eu-central-1 let aws_config_path = aws_config.1; Box::pin(async move { let middleware = - S3Middleware::new(Some(&aws_config_path), "default".into(), None).await; + S3Middleware::new(Some(aws_config_path), Some("default".into()), None); let presigned = middleware .generate_presigned_s3_request( diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 07340d35c..9f33b2866 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -114,7 +114,7 @@ async fn test_minio_download_repodata( minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), ) { - let middleware = S3Middleware::new(Some(&aws_config.1), "default".into(), Some(true)).await; + let middleware = S3Middleware::new(Some(aws_config.1), Some("default".into()), Some(true)); let download_client = Client::builder() .no_gzip() diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index ed0f80124..d8596d8e2 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -292,6 +292,380 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "649316840239f4e58df0b7f620c428f5fababbbca2d504488c641534050bd141" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f6f1124d6e19ab6daf7f2e615644305dc6cb2d706892a8a8c0b98db35de020" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.67.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc644164269a1e38ce7f2f7373629d3fb3d310c0e3feb5573a29744288b24d3" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb25f7129c74d36afe33405af4517524df8f74b635af8c2c8e91c1552b8397b2" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d03a3d5ef14851625eafd89660a751776f938bf32f309308b20dcca41c44b568" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3a9f073ae3a53b54421503063dfb87ff1ea83b876f567d92e8b8d9942ba91b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.2.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427cb637d15d63d6f9aae26358e1c9a9c09d5aa490d64b09354c8217cfef0f28" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05dd41a70fc74051758ee75b5c4db2c0ca070ed9229c3df50e9475cda1cb985" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.2.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ddc9bd6c28aeb303477170ddd183760a956a03e083b3902a990238a7e3792d" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.2.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -307,6 +681,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -319,6 +699,22 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "2.6.0" @@ -401,6 +797,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -517,6 +923,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -552,6 +964,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -586,6 +1007,28 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -674,6 +1117,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -738,12 +1191,44 @@ dependencies = [ "syn", ] +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "elsa" version = "1.10.0" @@ -872,6 +1357,16 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "file_url" version = "0.2.1" @@ -926,6 +1421,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1177,6 +1678,36 @@ dependencies = [ "async-trait", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.6" @@ -1188,7 +1719,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -1227,6 +1758,11 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1288,6 +1824,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.2.0" @@ -1299,6 +1846,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1306,7 +1864,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1317,8 +1875,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1328,7 +1886,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92baf25cf0b8c9246baecf3a444546360a97b569168fdf92563ee6a47829920c" dependencies = [ - "http", + "http 1.2.0", "http-serde", "reqwest", "serde", @@ -1341,7 +1899,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http", + "http 1.2.0", "serde", ] @@ -1351,6 +1909,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -1366,6 +1930,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.0" @@ -1375,9 +1963,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.6", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -1386,6 +1974,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -1393,14 +1997,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.0", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.16", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", "webpki-roots", ] @@ -1413,7 +2017,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.5.0", "hyper-util", "native-tls", "tokio", @@ -1430,9 +2034,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.0", "pin-project-lite", "socket2", "tokio", @@ -1865,6 +2469,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.0", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2192,6 +2805,23 @@ dependencies = [ "syn", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -2375,6 +3005,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -2615,7 +3255,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.16", "socket2", "thiserror 1.0.67", "tokio", @@ -2632,7 +3272,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls", + "rustls 0.23.16", "slab", "thiserror 1.0.67", "tinyvec", @@ -2700,7 +3340,7 @@ dependencies = [ [[package]] name = "rattler" -version = "0.28.8" +version = "0.28.9" dependencies = [ "anyhow", "console", @@ -2739,7 +3379,7 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "dashmap", @@ -2767,7 +3407,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.29.6" +version = "0.29.7" dependencies = [ "chrono", "dirs", @@ -2817,7 +3457,7 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.20.3" +version = "0.20.4" dependencies = [ "fs-err", "rattler_conda_types", @@ -2830,7 +3470,7 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.22.35" +version = "0.22.36" dependencies = [ "chrono", "file_url", @@ -2865,6 +3505,9 @@ version = "0.21.9" dependencies = [ "anyhow", "async-trait", + "aws-config", + "aws-runtime", + "aws-sdk-s3", "base64 0.22.1", "chrono", "dirs", @@ -2872,7 +3515,7 @@ dependencies = [ "getrandom", "google-cloud-auth", "google-cloud-token", - "http", + "http 1.2.0", "itertools 0.13.0", "keyring", "netrc-rs", @@ -2888,7 +3531,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.22.19" +version = "0.22.20" dependencies = [ "bzip2 0.5.0", "chrono", @@ -2924,7 +3567,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.21.28" +version = "0.21.29" dependencies = [ "anyhow", "async-compression", @@ -2940,7 +3583,7 @@ dependencies = [ "fs-err", "futures", "hex", - "http", + "http 1.2.0", "http-cache-semantics", "humansize", "humantime", @@ -2977,7 +3620,7 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.22.11" +version = "0.22.12" dependencies = [ "enum_dispatch", "fs-err", @@ -2993,7 +3636,7 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "1.3.0" +version = "1.3.1" dependencies = [ "chrono", "futures", @@ -3009,7 +3652,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "1.1.14" +version = "1.1.15" dependencies = [ "archspec", "libloading", @@ -3117,6 +3760,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -3135,12 +3784,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.6", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.0", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -3152,9 +3801,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.16", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -3162,7 +3811,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -3182,7 +3831,7 @@ checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" dependencies = [ "anyhow", "async-trait", - "http", + "http 1.2.0", "reqwest", "serde", "thiserror 1.0.67", @@ -3215,6 +3864,17 @@ dependencies = [ "rand", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.8" @@ -3264,6 +3924,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.42" @@ -3277,6 +3946,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.16" @@ -3286,11 +3967,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -3303,6 +3996,15 @@ dependencies = [ "security-framework 3.0.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3318,6 +4020,16 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3365,6 +4077,30 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secret-service" version = "4.0.0" @@ -3420,6 +4156,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" + [[package]] name = "serde" version = "1.0.216" @@ -3577,6 +4319,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -3674,6 +4426,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3915,6 +4677,7 @@ dependencies = [ "mio", "parking_lot", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.52.0", @@ -3941,13 +4704,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -4169,6 +4942,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -4617,6 +5396,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" diff --git a/py-rattler/Cargo.toml b/py-rattler/Cargo.toml index d88d20878..4a5a929e4 100644 --- a/py-rattler/Cargo.toml +++ b/py-rattler/Cargo.toml @@ -38,6 +38,7 @@ rattler_conda_types = { path = "../crates/rattler_conda_types", default-features rattler_digest = { path = "../crates/rattler_digest" } rattler_networking = { path = "../crates/rattler_networking", default-features = false, features = [ "gcs", + "s3", ] } rattler_shell = { path = "../crates/rattler_shell", default-features = false } rattler_virtual_packages = { path = "../crates/rattler_virtual_packages", default-features = false } diff --git a/py-rattler/rattler/networking/__init__.py b/py-rattler/rattler/networking/__init__.py index dd115384b..d669b7f67 100644 --- a/py-rattler/rattler/networking/__init__.py +++ b/py-rattler/rattler/networking/__init__.py @@ -1,5 +1,17 @@ from rattler.networking.client import Client -from rattler.networking.middleware import MirrorMiddleware, AuthenticationMiddleware, GCSMiddleware from rattler.networking.fetch_repo_data import fetch_repo_data +from rattler.networking.middleware import ( + AuthenticationMiddleware, + GCSMiddleware, + MirrorMiddleware, + S3Middleware, +) -__all__ = ["fetch_repo_data", "Client", "MirrorMiddleware", "AuthenticationMiddleware", "GCSMiddleware"] +__all__ = [ + "fetch_repo_data", + "Client", + "MirrorMiddleware", + "AuthenticationMiddleware", + "GCSMiddleware", + "S3Middleware", +] diff --git a/py-rattler/rattler/networking/client.py b/py-rattler/rattler/networking/client.py index 8aa361013..82cfce37c 100644 --- a/py-rattler/rattler/networking/client.py +++ b/py-rattler/rattler/networking/client.py @@ -1,6 +1,13 @@ from __future__ import annotations + +from rattler.networking.middleware import ( + AuthenticationMiddleware, + GCSMiddleware, + MirrorMiddleware, + OciMiddleware, + S3Middleware, +) from rattler.rattler import PyClientWithMiddleware -from rattler.networking.middleware import AuthenticationMiddleware, MirrorMiddleware, OciMiddleware, GCSMiddleware class Client: @@ -10,7 +17,9 @@ class Client: def __init__( self, - middlewares: list[AuthenticationMiddleware | MirrorMiddleware | OciMiddleware | GCSMiddleware] | None = None, + middlewares: ( + list[AuthenticationMiddleware | MirrorMiddleware | OciMiddleware | GCSMiddleware | S3Middleware] | None + ) = None, ) -> None: self._client = PyClientWithMiddleware( [middleware._middleware for middleware in middlewares] if middlewares else None diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index cbe652f09..006f0a594 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -1,5 +1,12 @@ from __future__ import annotations -from rattler.rattler import PyMirrorMiddleware, PyAuthenticationMiddleware, PyOciMiddleware, PyGCSMiddleware + +from rattler.rattler import ( + PyAuthenticationMiddleware, + PyGCSMiddleware, + PyMirrorMiddleware, + PyOciMiddleware, + PyS3Middleware, +) class MirrorMiddleware: @@ -107,6 +114,7 @@ class GCSMiddleware: GCSMiddleware() >>> Client([middleware]) Client() + ``` """ def __init__(self) -> None: @@ -114,3 +122,26 @@ def __init__(self) -> None: def __repr__(self) -> str: return f"{type(self).__name__}()" + + +class S3Middleware: + """ + Middleware to work with s3:// URLs + + Examples + -------- + ```python + >>> from rattler.networking import Client + >>> middleware = S3Middleware() + >>> middleware + S3Middleware() + >>> Client([middleware]) + Client() + ``` + """ + + def __init__(self) -> None: + self._middleware = PyS3Middleware() + + def __repr__(self) -> str: + return f"{type(self).__name__}()" diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index 904d2341a..da31c8e3e 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -53,6 +53,7 @@ use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, + PyS3Middleware, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; @@ -107,6 +108,7 @@ fn rattler<'py>(py: Python<'py>, m: Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; // Shell activation things diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 2ff7fce32..37230bdf1 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -1,7 +1,7 @@ use crate::networking::middleware::PyMiddleware; use pyo3::{pyclass, pymethods}; use rattler_networking::{ - AuthenticationMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, + AuthenticationMiddleware, GCSMiddleware, S3Middleware, MirrorMiddleware, OciMiddleware, }; use reqwest_middleware::ClientWithMiddleware; @@ -28,6 +28,7 @@ impl PyClientWithMiddleware { } PyMiddleware::Oci(middleware) => client.with(OciMiddleware::from(middleware)), PyMiddleware::Gcs(middleware) => client.with(GCSMiddleware::from(middleware)), + PyMiddleware::S3(middleware) => client.with(S3Middleware::from(middleware)), }); let client = client.build(); diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index cae3510c9..ccc183aa0 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -1,9 +1,9 @@ use pyo3::{pyclass, pymethods, FromPyObject, PyResult}; use rattler_networking::{ mirror_middleware::Mirror, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, - MirrorMiddleware, OciMiddleware, + MirrorMiddleware, OciMiddleware, S3Middleware, }; -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf}; use url::Url; use crate::error::PyRattlerError; @@ -14,6 +14,7 @@ pub enum PyMiddleware { Authentication(PyAuthenticationMiddleware), Oci(PyOciMiddleware), Gcs(PyGCSMiddleware), + S3(PyS3Middleware), } #[pyclass] @@ -113,3 +114,37 @@ impl From for GCSMiddleware { GCSMiddleware } } + +#[pyclass] +#[derive(Clone)] +pub struct PyS3Middleware { + pub(crate) config_file: Option, + pub(crate) profile: Option, + pub(crate) force_path_style: Option, +} + +#[pymethods] +impl PyS3Middleware { + #[new] + pub fn __init__( + config_file: Option, + profile: Option, + force_path_style: Option, + ) -> Self { + Self { + config_file, + profile, + force_path_style, + } + } +} + +impl From for S3Middleware { + fn from(_value: PyS3Middleware) -> Self { + S3Middleware::new( + _value.config_file.into(), + _value.profile.into(), + _value.force_path_style.into(), + ) + } +} From 18fee26a7447c0b7113a9e2cdfa7832aef221a34 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 19:11:08 +0100 Subject: [PATCH 08/77] fixes --- crates/rattler-bin/src/commands/create.rs | 2 +- pixi.toml | 2 +- py-rattler/rattler/networking/middleware.py | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index d7a093e76..14df69070 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -153,7 +153,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { authentication_storage, ))) .with(rattler_networking::OciMiddleware) - .with(rattler_networking::S3Middleware::new(None, None, None).await) + .with(rattler_networking::S3Middleware::new(None, None, None)) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/pixi.toml b/pixi.toml index 68c9e7a0f..6ea1c2257 100644 --- a/pixi.toml +++ b/pixi.toml @@ -24,7 +24,7 @@ cxx-compiler = "~=1.6.0" openssl = "~=3.1" make = "~=4.3" pkg-config = "~=0.29.2" -rust = "~=1.80.0" +rust = "~=1.81.0" cmake = "~=3.26.4" minio-server = "*" # minio-client = "*" # xref: https://github.com/conda-forge/staged-recipes/pull/28736 diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index 006f0a594..6fe8fbbf2 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -114,6 +114,7 @@ class GCSMiddleware: GCSMiddleware() >>> Client([middleware]) Client() + >>> ``` """ @@ -137,6 +138,7 @@ class S3Middleware: S3Middleware() >>> Client([middleware]) Client() + >>> ``` """ From 56a1d5d43586228d9145ec6048a5af9a62191dd3 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 19:25:00 +0100 Subject: [PATCH 09/77] . --- py-rattler/rattler/networking/middleware.py | 7 +++++-- py-rattler/src/networking/client.rs | 2 +- py-rattler/src/networking/middleware.rs | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index 6fe8fbbf2..db76f7fad 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -1,4 +1,5 @@ from __future__ import annotations +from pathlib import Path from rattler.rattler import ( PyAuthenticationMiddleware, @@ -142,8 +143,10 @@ class S3Middleware: ``` """ - def __init__(self) -> None: - self._middleware = PyS3Middleware() + def __init__( + self, config_file: Path | None = None, profile: str | None = None, force_path_style: bool | None = None + ) -> None: + self._middleware = PyS3Middleware(config_file, profile, force_path_style) def __repr__(self) -> str: return f"{type(self).__name__}()" diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index 37230bdf1..8477a8927 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -1,7 +1,7 @@ use crate::networking::middleware::PyMiddleware; use pyo3::{pyclass, pymethods}; use rattler_networking::{ - AuthenticationMiddleware, GCSMiddleware, S3Middleware, MirrorMiddleware, OciMiddleware, + AuthenticationMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, S3Middleware, }; use reqwest_middleware::ClientWithMiddleware; diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index ccc183aa0..5c15e9fff 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -126,6 +126,7 @@ pub struct PyS3Middleware { #[pymethods] impl PyS3Middleware { #[new] + #[pyo3(signature = (config_file=None, profile=None, force_path_style=None))] pub fn __init__( config_file: Option, profile: Option, From 592139a6e667b3fc42b11661789364e4635e61f0 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 19:28:32 +0100 Subject: [PATCH 10/77] revert bad formatting --- .github/workflows/rust-compile.yml | 89 +++++++++--------------------- 1 file changed, 26 insertions(+), 63 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 6a1674853..5c8440486 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -59,65 +59,25 @@ jobs: fail-fast: false matrix: include: - - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-22.04 } - - { - name: "Linux-aarch64", - target: aarch64-unknown-linux-musl, - os: ubuntu-latest, - skip-tests: true, - } - - { - name: "Linux-arm", - target: arm-unknown-linux-musleabi, - os: ubuntu-latest, - use-cross: true, - skip-tests: true, - } - + - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-22.04 } + - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } + - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - - { - name: "Linux-powerpc64", - target: powerpc64-unknown-linux-gnu, - os: ubuntu-latest, - use-cross: true, - skip-tests: true, - } - - { - name: "Linux-powerpc64le", - target: powerpc64le-unknown-linux-gnu, - os: ubuntu-latest, - use-cross: true, - skip-tests: true, - } - - - { - name: "Linux-s390x", - target: s390x-unknown-linux-gnu, - os: ubuntu-latest, - use-cross: true, - skip-tests: true, - } - - - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { - name: "macOS-aarch64", - target: aarch64-apple-darwin, - os: macOS-latest, - skip-tests: true, - } - - - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - - { - name: "Windows-aarch64", - target: aarch64-pc-windows-msvc, - os: windows-latest, - skip-tests: true, - } + # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + + - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + + - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } + - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } + + - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } + - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } steps: - name: Checkout source code uses: actions/checkout@v4 @@ -154,17 +114,17 @@ jobs: - name: Use rustls on musl targets. id: build-options - if: - contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc') || - startsWith(matrix.target, 's390x') + if: contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc') || startsWith(matrix.target, 's390x') run: | echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT - name: Build run: > - cargo build --all-targets - --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} ${{ - steps.build-options.outputs.CARGO_BUILD_OPTIONS}} + cargo build + --all-targets + --features ${{ env.DEFAULT_FEATURES }} + --target ${{ matrix.target }} + ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} - name: Disable testing the tools crate if cross compiling id: test-options @@ -183,9 +143,12 @@ jobs: env: GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }} run: > - cargo nextest run --workspace --features ${{ env.DEFAULT_FEATURES }} --target ${{ - matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} ${{ - steps.test-options.outputs.CARGO_TEST_OPTIONS}} + cargo nextest run + --workspace + --features ${{ env.DEFAULT_FEATURES }} + --target ${{ matrix.target }} + ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} + ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} - name: Run doctests if: ${{ !matrix.skip-tests }} From f64d091803241dc139c11162a616e13f01ebdbe7 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 19:52:58 +0100 Subject: [PATCH 11/77] fix --- crates/rattler_networking/tests/integration_test.rs | 8 ++++---- py-rattler/src/networking/middleware.rs | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 9f33b2866..93006b866 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -10,6 +10,7 @@ use tempfile::{tempdir, TempDir}; struct MinioServer { handle: std::process::Child, + #[allow(dead_code)] directory: tempfile::TempDir, } @@ -23,7 +24,7 @@ impl MinioServer { "127.0.0.1:9000", ]; let handle = std::process::Command::new("minio") - .args(&args) + .args(args) .spawn() .expect("Failed to start Minio server"); eprintln!( @@ -49,9 +50,7 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p command.env(key, value); } let output = command.output().expect("Failed to run command"); - if !output.status.success() { - eprintln!("Command failed: {:?}", output); - } + assert!(output.status.success()); output } @@ -111,6 +110,7 @@ region = eu-central-1 #[tokio::test] #[serial] async fn test_minio_download_repodata( + #[allow(unused_variables)] minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), ) { diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 5c15e9fff..d3ec6f260 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -143,9 +143,9 @@ impl PyS3Middleware { impl From for S3Middleware { fn from(_value: PyS3Middleware) -> Self { S3Middleware::new( - _value.config_file.into(), - _value.profile.into(), - _value.force_path_style.into(), + _value.config_file, + _value.profile, + _value.force_path_style, ) } } From 22205e8686e9de639e1c33974b8e9a3cc6878e56 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 19:57:19 +0100 Subject: [PATCH 12/77] fix --- crates/rattler_networking/tests/integration_test.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 93006b866..e25060569 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -110,8 +110,7 @@ region = eu-central-1 #[tokio::test] #[serial] async fn test_minio_download_repodata( - #[allow(unused_variables)] - minio_server: MinioServer, + #[allow(unused_variables)] minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), ) { let middleware = S3Middleware::new(Some(aws_config.1), Some("default".into()), Some(true)); From c5052b36ad0c7306d0c4a3c84b7463726ce312ef Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 20:02:03 +0100 Subject: [PATCH 13/77] fix --- py-rattler/src/networking/middleware.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index d3ec6f260..d28508f8e 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -142,10 +142,6 @@ impl PyS3Middleware { impl From for S3Middleware { fn from(_value: PyS3Middleware) -> Self { - S3Middleware::new( - _value.config_file, - _value.profile, - _value.force_path_style, - ) + S3Middleware::new(_value.config_file, _value.profile, _value.force_path_style) } } From c3c5689b800a664110aa8bc66cb8d7170b0e74bd Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 20:09:09 +0100 Subject: [PATCH 14/77] install minio-server and client --- .github/workflows/rust-compile.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 5c8440486..e9822fdaf 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -138,6 +138,16 @@ jobs: with: tool: cargo-nextest + - name: Set up pixi + uses: prefix-dev/setup-pixi@v0.8.1 + with: + run-install: false + + - name: Install minio + run: | + pixi global install minio-server + pixi global install minio-client + - name: Run tests if: ${{ !matrix.skip-tests }} env: From bdb4f68a340a16a6e6155a48684763e1bf4ade59 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 2 Jan 2025 20:35:00 +0100 Subject: [PATCH 15/77] only install when testing --- .github/workflows/rust-compile.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index e9822fdaf..b32501825 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -139,11 +139,13 @@ jobs: tool: cargo-nextest - name: Set up pixi + if: ${{ !matrix.skip-tests }} uses: prefix-dev/setup-pixi@v0.8.1 with: run-install: false - name: Install minio + if: ${{ !matrix.skip-tests }} run: | pixi global install minio-server pixi global install minio-client From 0d7927261f32319fd3c5234474c4d03dd0b52010 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 10:26:06 +0100 Subject: [PATCH 16/77] Add s3 to allowed url schemes --- crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs index 8a101659c..85a9a3e18 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs @@ -54,6 +54,7 @@ impl<'g> SubdirBuilder<'g> { || url.scheme() == "https" || url.scheme() == "gcs" || url.scheme() == "oci" + || url.scheme() == "s3" { let source_config = self.gateway.channel_config.get(&self.channel.base_url); From 604c0c81bd97ffee5a0f01ff9a12d3e1f506a269 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 10:52:27 +0100 Subject: [PATCH 17/77] Fix leading slash issue --- crates/rattler_networking/src/s3_middleware.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 36035b958..2623231c8 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -97,7 +97,12 @@ impl Middleware for S3Middleware { if req.url().scheme() == "s3" { let url = req.url().clone(); let bucket_name = url.host_str().expect("Host should be present in S3 URL"); - let key = url.path(); + let key = url.path().strip_prefix("/").ok_or_else(|| { + reqwest_middleware::Error::middleware(std::io::Error::new( + std::io::ErrorKind::Other, + "Missing prefix", + )) + })?; let presigned_request = self.generate_presigned_s3_request(bucket_name, key).await?; *req.url_mut() = Url::parse(presigned_request.uri()) From 97421eea785382165812a694e370cd38915513f9 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 11:28:24 +0100 Subject: [PATCH 18/77] add minio-client --- pixi.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixi.toml b/pixi.toml index 6ea1c2257..62dc7bc5f 100644 --- a/pixi.toml +++ b/pixi.toml @@ -27,7 +27,7 @@ pkg-config = "~=0.29.2" rust = "~=1.81.0" cmake = "~=3.26.4" minio-server = "*" -# minio-client = "*" # xref: https://github.com/conda-forge/staged-recipes/pull/28736 +minio-client = "*" [target.linux-64.dependencies] clang = ">=18.1.8,<19.0" From cbf2cc3d1196d0767d9377fff3863234531f336b Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 14:43:02 +0100 Subject: [PATCH 19/77] different api --- crates/rattler-bin/src/commands/create.rs | 2 +- .../src/authentication_middleware.rs | 3 + .../authentication_storage/authentication.rs | 9 + .../rattler_networking/src/s3_middleware.rs | 203 ++++++++---------- .../tests/integration_test.rs | 13 +- 5 files changed, 111 insertions(+), 119 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 14df69070..570a7b110 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -153,7 +153,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { authentication_storage, ))) .with(rattler_networking::OciMiddleware) - .with(rattler_networking::S3Middleware::new(None, None, None)) + .with(rattler_networking::S3Middleware::new(None)) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 25e2a3af4..df1f88799 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -106,6 +106,9 @@ impl AuthenticationMiddleware { Ok(req) } Authentication::CondaToken(_) => Ok(req), + Authentication::S3Credentials { .. } => { + panic!("S3 credentials should be handled by the S3 middleware") + } // todo } } else { Ok(req) diff --git a/crates/rattler_networking/src/authentication_storage/authentication.rs b/crates/rattler_networking/src/authentication_storage/authentication.rs index 6939b6675..74f9f3128 100644 --- a/crates/rattler_networking/src/authentication_storage/authentication.rs +++ b/crates/rattler_networking/src/authentication_storage/authentication.rs @@ -19,6 +19,15 @@ pub enum Authentication { }, /// A conda token is sent in the URL as `/t/{TOKEN}/...` CondaToken(String), + /// S3 credentials + S3Credentials { + /// The access key ID to use for S3 authentication + access_key_id: String, + /// The secret access key to use for S3 authentication + secret_access_key: String, + /// The session token to use for S3 authentication + session_token: Option, + }, } /// An error that can occur when parsing an authentication string diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 2623231c8..05c4eb758 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -1,6 +1,4 @@ //! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket -use std::path::PathBuf; - use async_trait::async_trait; use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; @@ -8,72 +6,91 @@ use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; use url::Url; +use crate::{Authentication, AuthenticationStorage}; + /// S3 middleware to authenticate requests. pub struct S3Middleware { - config_file: Option, - profile: Option, - force_path_style: Option, + config: Option, expiration: std::time::Duration, } -async fn create_client( - config_file: Option, - profile: Option, - force_path_style: Option, -) -> aws_sdk_s3::Client { - let mut aws_config_builder = aws_config::defaults(BehaviorVersion::latest()); - if let Some(config_file) = config_file { - let mut builder = aws_runtime::env_config::file::EnvConfigFiles::builder(); - builder = builder.with_file( - aws_runtime::env_config::file::EnvConfigFileKind::Config, - config_file, - ); - let env_config_files = builder.build(); - aws_config_builder = aws_config_builder.profile_files(env_config_files); - } - - if let Some(profile) = profile { - aws_config_builder = aws_config_builder.profile_name(profile); - }; - let sdk_config = aws_config_builder.load().await; +/// Configuration for the S3 middleware. +#[derive(Clone)] +pub struct S3Config { + /// The authentication storage to use for the S3 client. + pub auth_storage: AuthenticationStorage, + /// The endpoint URL to use for the S3 client. + pub endpoint_url: Url, + /// The region to use for the S3 client. + pub region: String, + /// Whether to force path style for the S3 client. + pub force_path_style: bool, +} - let mut builder = aws_sdk_s3::config::Builder::from(&sdk_config); - if let Some(force_path_style) = force_path_style { - builder = builder.force_path_style(force_path_style); - }; - let s3_config = builder.build(); +async fn create_client(config: Option, url: Url) -> aws_sdk_s3::Client { + match config { + Some(config) => { + let auth = config.auth_storage.get_by_url(url).unwrap(); // todo + let config_builder = match auth { + ( + _, + Some(Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + }), + ) => aws_sdk_s3::config::Builder::new() + .endpoint_url(config.endpoint_url) + .region(aws_sdk_s3::config::Region::new(config.region)) + .force_path_style(config.force_path_style) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "pixi", + )), + (_, Some(_)) => { + panic!("Unsupported authentication method"); // todo proper error message + } + (_, None) => aws_sdk_s3::config::Builder::new() + .endpoint_url(config.endpoint_url) + .region(aws_sdk_s3::config::Region::new(config.region)) + .force_path_style(config.force_path_style), + }; - let client: aws_sdk_s3::Client = aws_sdk_s3::Client::from_conf(s3_config); - client + let aws_config = config_builder.build(); + aws_sdk_s3::Client::from_conf(aws_config) + } + None => { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); + aws_sdk_s3::Client::from_conf(config) + } + } } impl S3Middleware { /// Create a new S3 middleware. - pub fn new( - config_file: Option, - profile: Option, - force_path_style: Option, - ) -> Self { + pub fn new(config: Option) -> Self { Self { - config_file, - profile, - force_path_style, + config, expiration: std::time::Duration::from_secs(300), } } /// Generate a presigned S3 `GetObject` request. - async fn generate_presigned_s3_request( - &self, - bucket_name: &str, - key: &str, - ) -> MiddlewareResult { - let client = create_client( - self.config_file.clone(), - self.profile.clone(), - self.force_path_style, - ) - .await; + async fn generate_presigned_s3_request(&self, url: Url) -> MiddlewareResult { + let client = create_client(self.config.clone(), url.clone()).await; + + let bucket_name = url.host_str().expect("Host should be present in S3 URL"); + let key = url.path().strip_prefix("/").ok_or_else(|| { + reqwest_middleware::Error::middleware(std::io::Error::new( + std::io::ErrorKind::Other, + "Missing prefix", + )) + })?; + let builder = client.get_object().bucket(bucket_name).key(key); builder .presigned( @@ -96,14 +113,7 @@ impl Middleware for S3Middleware { ) -> MiddlewareResult { if req.url().scheme() == "s3" { let url = req.url().clone(); - let bucket_name = url.host_str().expect("Host should be present in S3 URL"); - let key = url.path().strip_prefix("/").ok_or_else(|| { - reqwest_middleware::Error::middleware(std::io::Error::new( - std::io::ErrorKind::Other, - "Missing prefix", - )) - })?; - let presigned_request = self.generate_presigned_s3_request(bucket_name, key).await?; + let presigned_request = self.generate_presigned_s3_request(url).await?; *req.url_mut() = Url::parse(presigned_request.uri()) .map_err(reqwest_middleware::Error::middleware)?; @@ -136,28 +146,28 @@ mod tests { #[tokio::test] #[serial] - async fn test_presigned_s3_request() { + async fn test_presigned_s3_request_endpoint_url() { with_env( HashMap::from([ ("AWS_ACCESS_KEY_ID", "minioadmin"), ("AWS_SECRET_ACCESS_KEY", "minioadmin"), ("AWS_REGION", "eu-central-1"), - ("AWS_ENDPOINT_URL", "http://localhost:9000"), + ("AWS_ENDPOINT_URL", "http://custom-aws"), ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None, None, Some(true)); + let middleware = S3Middleware::new(None); let presigned = middleware .generate_presigned_s3_request( - "rattler-s3-testing", - "my-channel/noarch/repodata.json", + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .unwrap(), ) .await .unwrap(); assert!( presigned.uri().to_string().starts_with( - "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?" + "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" ), "Unexpected presigned URL: {:?}", presigned.uri() @@ -179,12 +189,11 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None, None, None); + let middleware = S3Middleware::new(None); let presigned = middleware .generate_presigned_s3_request( - "rattler-s3-testing", - "my-channel/noarch/repodata.json", + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap() ) .await .unwrap(); @@ -221,44 +230,6 @@ region = eu-central-1 (temp_dir, aws_config_path) } - #[rstest] - #[tokio::test] - #[serial] - async fn test_presigned_s3_request_custom_config(aws_config: (TempDir, std::path::PathBuf)) { - let middleware = S3Middleware::new(Some(aws_config.1), None, None); - - let presigned = middleware - .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") - .await - .unwrap(); - assert!( - presigned.uri().to_string().starts_with( - "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" - ), - "Unexpected presigned URL: {:?}", - presigned.uri() - ); - } - - #[rstest] - #[tokio::test] - #[serial] - async fn test_presigned_s3_request_different_profile( - aws_config: (TempDir, std::path::PathBuf), - ) { - let middleware = S3Middleware::new(Some(aws_config.1), Some("packages".into()), None); - - let presigned = middleware - .generate_presigned_s3_request("rattler-s3-testing", "my-channel/noarch/repodata.json") - .await - .unwrap(); - assert!( - presigned.uri().to_string().contains("localhost:8000"), - "Unexpected presigned URL: {:?}", - presigned.uri() - ); - } - #[rstest] #[tokio::test] #[serial] @@ -272,12 +243,12 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(None, None, None); + let middleware = S3Middleware::new(None); let presigned = middleware .generate_presigned_s3_request( - "rattler-s3-testing", - "my-channel/noarch/repodata.json", + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .unwrap(), ) .await .unwrap(); @@ -292,23 +263,23 @@ region = eu-central-1 .await; } - /// Test that environment variables take precedence over the configuration file. #[rstest] #[tokio::test] #[serial] async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { with_env( - HashMap::from([("AWS_ENDPOINT_URL", "http://localhost:9000")]), + HashMap::from([ + ("AWS_ENDPOINT_URL", "http://localhost:9000"), + ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), + ]), move || { - let aws_config_path = aws_config.1; Box::pin(async move { - let middleware = - S3Middleware::new(Some(aws_config_path), Some("default".into()), None); + let middleware = S3Middleware::new(None); let presigned = middleware .generate_presigned_s3_request( - "rattler-s3-testing", - "my-channel/noarch/repodata.json", + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .unwrap(), ) .await .unwrap(); diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index e25060569..9283aa165 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,10 +1,13 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; -use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage, S3Middleware}; +use rattler_networking::{ + s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware, +}; use reqwest::Client; use rstest::*; use serial_test::serial; use tempfile::{tempdir, TempDir}; +use url::Url; /* ----------------------------------- MINIO UTILS ---------------------------------- */ @@ -113,7 +116,13 @@ async fn test_minio_download_repodata( #[allow(unused_variables)] minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), ) { - let middleware = S3Middleware::new(Some(aws_config.1), Some("default".into()), Some(true)); + // TODO: also test with_env + let middleware = S3Middleware::new(Some(S3Config { + auth_storage: AuthenticationStorage::default(), // todo: create custom storage here + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + })); let download_client = Client::builder() .no_gzip() From 4d8dfc239b820ce7755bebdcad35a47b56adb499 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 15:25:39 +0100 Subject: [PATCH 20/77] adjust py-rattler --- py-rattler/rattler/networking/middleware.py | 36 +++++++++++- py-rattler/src/lib.rs | 3 +- py-rattler/src/networking/middleware.rs | 63 +++++++++++++++------ 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index db76f7fad..c6800ff95 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -7,6 +7,7 @@ PyMirrorMiddleware, PyOciMiddleware, PyS3Middleware, + PyS3Config, ) @@ -126,6 +127,34 @@ def __repr__(self) -> str: return f"{type(self).__name__}()" +class S3Config: + """ + Middleware to work with s3:// URLs + + Examples + -------- + ```python + >>> from rattler.networking import S3Middleware + >>> config = S3Config("http://localhost:9000", "eu-central-1", True) + >>> config + S3Config(http://localhost:9000, eu-central-1, True) + >>> middleware = S3Middleware(config) + >>> middleware + S3Middleware() + >>> + ``` + """ + + def __init__(self, endpoint_url: str, region: str, force_path_style: bool) -> None: + self._config = PyS3Config(endpoint_url, region, force_path_style) + self.endpoint_url = endpoint_url + self.region = region + self.force_path_style = force_path_style + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.endpoint_url}, {self.region}, {self.force_path_style})" + + class S3Middleware: """ Middleware to work with s3:// URLs @@ -144,9 +173,12 @@ class S3Middleware: """ def __init__( - self, config_file: Path | None = None, profile: str | None = None, force_path_style: bool | None = None + self, config: S3Config | None = None ) -> None: - self._middleware = PyS3Middleware(config_file, profile, force_path_style) + if config is not None: + self._middleware = PyS3Middleware(config._config) + else: + self._middleware = PyS3Middleware() def __repr__(self) -> str: return f"{type(self).__name__}()" diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index da31c8e3e..90b3daad9 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -53,7 +53,7 @@ use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, - PyS3Middleware, + PyS3Middleware, PyS3Config, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; @@ -109,6 +109,7 @@ fn rattler<'py>(py: Python<'py>, m: Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; // Shell activation things diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index d28508f8e..0c29de901 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -1,9 +1,8 @@ use pyo3::{pyclass, pymethods, FromPyObject, PyResult}; use rattler_networking::{ - mirror_middleware::Mirror, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, - MirrorMiddleware, OciMiddleware, S3Middleware, + mirror_middleware::Mirror, s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, OciMiddleware, S3Middleware }; -use std::{collections::HashMap, path::PathBuf}; +use std::collections::HashMap; use url::Url; use crate::error::PyRattlerError; @@ -117,31 +116,61 @@ impl From for GCSMiddleware { #[pyclass] #[derive(Clone)] -pub struct PyS3Middleware { - pub(crate) config_file: Option, - pub(crate) profile: Option, - pub(crate) force_path_style: Option, +pub struct PyS3Config { + pub(crate) endpoint_url: Url, + pub(crate) region: String, + pub(crate) force_path_style: bool, } #[pymethods] -impl PyS3Middleware { +impl PyS3Config { #[new] - #[pyo3(signature = (config_file=None, profile=None, force_path_style=None))] pub fn __init__( - config_file: Option, - profile: Option, - force_path_style: Option, - ) -> Self { - Self { - config_file, - profile, + endpoint_url: String, + region: String, + force_path_style: bool, + ) -> PyResult { + Ok(Self { + endpoint_url: Url::parse(&endpoint_url).map_err(PyRattlerError::from)?, + region, force_path_style, + }) + } +} + +impl From for S3Config { + fn from(_value: PyS3Config) -> Self { + S3Config { + auth_storage: AuthenticationStorage::default(), + endpoint_url: _value.endpoint_url, + region: _value.region, + force_path_style: _value.force_path_style, } } } +#[pyclass] +#[derive(Clone)] +pub struct PyS3Middleware { + pub(crate) s3_config: Option, +} + +#[pymethods] +impl PyS3Middleware { + #[new] + #[pyo3(signature = (s3_config=None))] + pub fn __init__( + s3_config: Option, + ) -> PyResult { + Ok(Self { s3_config }) + } +} + impl From for S3Middleware { fn from(_value: PyS3Middleware) -> Self { - S3Middleware::new(_value.config_file, _value.profile, _value.force_path_style) + match _value.s3_config { + Some(config) => S3Middleware::new(Some(config.into())), + None => S3Middleware::new(None), + } } } From c59a3f40a19e833da8c33685093930206bde0553 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 15:45:01 +0100 Subject: [PATCH 21/77] Make client creation public --- crates/rattler_networking/src/s3_middleware.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 05c4eb758..813432b89 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -27,7 +27,8 @@ pub struct S3Config { pub force_path_style: bool, } -async fn create_client(config: Option, url: Url) -> aws_sdk_s3::Client { +/// Create an S3 client with the given configuration. +pub async fn create_s3_client(config: Option, url: Url) -> aws_sdk_s3::Client { match config { Some(config) => { let auth = config.auth_storage.get_by_url(url).unwrap(); // todo @@ -81,7 +82,7 @@ impl S3Middleware { /// Generate a presigned S3 `GetObject` request. async fn generate_presigned_s3_request(&self, url: Url) -> MiddlewareResult { - let client = create_client(self.config.clone(), url.clone()).await; + let client = create_s3_client(self.config.clone(), url.clone()).await; let bucket_name = url.host_str().expect("Host should be present in S3 URL"); let key = url.path().strip_prefix("/").ok_or_else(|| { From 9838334d8a0cdc408ff647fc46e35c28d4784506 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 17:13:00 +0100 Subject: [PATCH 22/77] Add todo to infer path style --- crates/rattler_networking/src/s3_middleware.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 813432b89..9db31f0a9 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -15,7 +15,7 @@ pub struct S3Middleware { } /// Configuration for the S3 middleware. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct S3Config { /// The authentication storage to use for the S3 client. pub auth_storage: AuthenticationStorage, @@ -65,6 +65,8 @@ pub async fn create_s3_client(config: Option, url: Url) -> aws_sdk_s3: } None => { let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + // TODO: infer path style from endpoint URL or other means and set + // .force_path_style(true) on builder if necessary let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); aws_sdk_s3::Client::from_conf(config) } From f8fde26ac7a87fd7c8f3b9bac1ac9f474776a15e Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 3 Jan 2025 18:18:12 +0100 Subject: [PATCH 23/77] Fix integration test and authstorage lookup --- crates/rattler_networking/Cargo.toml | 2 +- .../src/authentication_middleware.rs | 5 +- .../src/authentication_storage/storage.rs | 26 ++++++ .../rattler_networking/src/s3_middleware.rs | 84 +++++++++---------- .../tests/integration_test.rs | 39 ++++----- 5 files changed, 86 insertions(+), 70 deletions(-) diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 5ee094053..1798ccf3b 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls"] +default = ["native-tls", "s3"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index df1f88799..79b6eabcb 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -105,10 +105,7 @@ impl AuthenticationMiddleware { .insert(reqwest::header::AUTHORIZATION, header_value); Ok(req) } - Authentication::CondaToken(_) => Ok(req), - Authentication::S3Credentials { .. } => { - panic!("S3 credentials should be handled by the S3 middleware") - } // todo + Authentication::CondaToken(_) | Authentication::S3Credentials { .. } => Ok(req), } } else { Ok(req) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 12a983ced..1f8176207 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -159,6 +159,32 @@ impl AuthenticationStorage { Ok(Some(credentials)) => return Ok((url, Some(credentials))), }; + // S3 protocol URLs need to be treated separately since they follow a different schema + if url.scheme() == "s3" { + let mut current_url = url.clone(); + loop { + match self.get(current_url.as_str()) { + Ok(None) => { + let possible_rest = + current_url.as_str().rsplit_once('/').map(|(rest, _)| rest); + + match possible_rest { + Some(rest) => { + if let Ok(new_url) = Url::parse(rest) { + current_url = new_url; + } else { + return Ok((url, None)); + } + } + _ => return Ok((url, None)), // No more subdomains to check + } + } + Ok(Some(credentials)) => return Ok((url, Some(credentials))), + Err(_) => return Ok((url, None)), + } + } + } + // Check for credentials under e.g. `*.prefix.dev` let Some(mut domain) = url.domain() else { return Ok((url, None)); diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 9db31f0a9..cbf481a7d 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -27,49 +27,45 @@ pub struct S3Config { pub force_path_style: bool, } -/// Create an S3 client with the given configuration. -pub async fn create_s3_client(config: Option, url: Url) -> aws_sdk_s3::Client { - match config { - Some(config) => { - let auth = config.auth_storage.get_by_url(url).unwrap(); // todo - let config_builder = match auth { - ( - _, - Some(Authentication::S3Credentials { - access_key_id, - secret_access_key, - session_token, - }), - ) => aws_sdk_s3::config::Builder::new() - .endpoint_url(config.endpoint_url) - .region(aws_sdk_s3::config::Region::new(config.region)) - .force_path_style(config.force_path_style) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - access_key_id, - secret_access_key, - session_token, - None, - "pixi", - )), - (_, Some(_)) => { - panic!("Unsupported authentication method"); // todo proper error message - } - (_, None) => aws_sdk_s3::config::Builder::new() - .endpoint_url(config.endpoint_url) - .region(aws_sdk_s3::config::Region::new(config.region)) - .force_path_style(config.force_path_style), - }; - - let aws_config = config_builder.build(); - aws_sdk_s3::Client::from_conf(aws_config) - } - None => { - let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; - // TODO: infer path style from endpoint URL or other means and set - // .force_path_style(true) on builder if necessary - let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); - aws_sdk_s3::Client::from_conf(config) - } +/// Create an S3 client for given channel with provided configuration. +pub async fn create_s3_client(config: Option, url: Option) -> aws_sdk_s3::Client { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + if let (Some(config), Some(url)) = (config, url) { + let auth = config.auth_storage.get_by_url(url).unwrap(); // todo + let config_builder = match auth { + ( + _, + Some(Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + }), + ) => aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(config.endpoint_url) + .region(aws_sdk_s3::config::Region::new(config.region)) + .force_path_style(config.force_path_style) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "pixi", + )), + (_, Some(_)) => { + panic!("Unsupported authentication method"); // todo proper error message + } + (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(config.endpoint_url) + .region(aws_sdk_s3::config::Region::new(config.region)) + .force_path_style(config.force_path_style), + }; + let aws_config = config_builder.build(); + aws_sdk_s3::Client::from_conf(aws_config) + } else { + // TODO: infer path style from endpoint URL or other means and set + // .force_path_style(true) on builder if necessary + let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); + aws_sdk_s3::Client::from_conf(config) } } @@ -84,7 +80,7 @@ impl S3Middleware { /// Generate a presigned S3 `GetObject` request. async fn generate_presigned_s3_request(&self, url: Url) -> MiddlewareResult { - let client = create_s3_client(self.config.clone(), url.clone()).await; + let client = create_s3_client(self.config.clone(), Some(url.clone())).await; let bucket_name = url.host_str().expect("Host should be present in S3 URL"); let key = url.path().strip_prefix("/").ok_or_else(|| { diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 9283aa165..110a59285 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -95,18 +95,21 @@ fn minio_server() -> MinioServer { } #[fixture] -fn aws_config() -> (TempDir, std::path::PathBuf) { +fn auth_file() -> (TempDir, std::path::PathBuf) { let temp_dir = tempdir().unwrap(); let aws_config = r#" -[profile default] -aws_access_key_id = minioadmin -aws_secret_access_key = minioadmin -endpoint_url = http://localhost:9000 -region = eu-central-1 +{ + "s3://rattler-s3-testing/my-channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } +} "#; - let aws_config_path = temp_dir.path().join("aws.config"); - std::fs::write(&aws_config_path, aws_config).unwrap(); - (temp_dir, aws_config_path) + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, aws_config).unwrap(); + (temp_dir, credentials_path) } #[rstest] @@ -114,26 +117,20 @@ region = eu-central-1 #[serial] async fn test_minio_download_repodata( #[allow(unused_variables)] minio_server: MinioServer, - aws_config: (TempDir, std::path::PathBuf), + auth_file: (TempDir, std::path::PathBuf), ) { - // TODO: also test with_env + // TODO: also test with AWS environment variables + let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); let middleware = S3Middleware::new(Some(S3Config { - auth_storage: AuthenticationStorage::default(), // todo: create custom storage here + auth_storage: auth_storage.clone(), endpoint_url: Url::parse("http://localhost:9000").unwrap(), region: "eu-central-1".into(), force_path_style: true, })); - let download_client = Client::builder() - .no_gzip() - .build() - .expect("failed to create client"); - - let authentication_storage = AuthenticationStorage::default(); + let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new( - authentication_storage, - ))) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) .with(middleware) .build(); From e095b8ec0d7462188bd75433f9857ddfc63f0fba Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 18:25:13 +0100 Subject: [PATCH 24/77] make enum instead of option --- .../rattler_networking/src/s3_middleware.rs | 70 +++++++++++-------- .../tests/integration_test.rs | 7 +- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index cbf481a7d..70bbc817d 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -8,30 +8,44 @@ use url::Url; use crate::{Authentication, AuthenticationStorage}; +/// Configuration for the S3 middleware. +#[derive(Clone)] +pub enum S3Config { + /// Use the default AWS configuration. + FromAWS, + /// Use a custom configuration. + Custom { + /// The authentication storage to use for the S3 client. + auth_storage: AuthenticationStorage, + /// The endpoint URL to use for the S3 client. + endpoint_url: Url, + /// The region to use for the S3 client. + region: String, + /// Whether to force path style for the S3 client. + force_path_style: bool, + }, +} + /// S3 middleware to authenticate requests. pub struct S3Middleware { - config: Option, + config: S3Config, expiration: std::time::Duration, } -/// Configuration for the S3 middleware. -#[derive(Clone, Debug)] -pub struct S3Config { - /// The authentication storage to use for the S3 client. - pub auth_storage: AuthenticationStorage, - /// The endpoint URL to use for the S3 client. - pub endpoint_url: Url, - /// The region to use for the S3 client. - pub region: String, - /// Whether to force path style for the S3 client. - pub force_path_style: bool, -} - /// Create an S3 client for given channel with provided configuration. -pub async fn create_s3_client(config: Option, url: Option) -> aws_sdk_s3::Client { +pub async fn create_s3_client(config: S3Config, url: Option) -> aws_sdk_s3::Client { let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; - if let (Some(config), Some(url)) = (config, url) { - let auth = config.auth_storage.get_by_url(url).unwrap(); // todo + if let ( + S3Config::Custom { + endpoint_url, + region, + force_path_style, + auth_storage, + }, + Some(url), + ) = (config, url) + { + let auth = auth_storage.get_by_url(url).unwrap(); // todo let config_builder = match auth { ( _, @@ -41,9 +55,9 @@ pub async fn create_s3_client(config: Option, url: Option) -> aws session_token, }), ) => aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(config.endpoint_url) - .region(aws_sdk_s3::config::Region::new(config.region)) - .force_path_style(config.force_path_style) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) .credentials_provider(aws_sdk_s3::config::Credentials::new( access_key_id, secret_access_key, @@ -55,9 +69,9 @@ pub async fn create_s3_client(config: Option, url: Option) -> aws panic!("Unsupported authentication method"); // todo proper error message } (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(config.endpoint_url) - .region(aws_sdk_s3::config::Region::new(config.region)) - .force_path_style(config.force_path_style), + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style), }; let aws_config = config_builder.build(); aws_sdk_s3::Client::from_conf(aws_config) @@ -71,7 +85,7 @@ pub async fn create_s3_client(config: Option, url: Option) -> aws impl S3Middleware { /// Create a new S3 middleware. - pub fn new(config: Option) -> Self { + pub fn new(config: S3Config) -> Self { Self { config, expiration: std::time::Duration::from_secs(300), @@ -155,7 +169,7 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None); + let middleware = S3Middleware::new(S3Config::FromAWS); let presigned = middleware .generate_presigned_s3_request( @@ -188,7 +202,7 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(None); + let middleware = S3Middleware::new(S3Config::FromAWS); let presigned = middleware .generate_presigned_s3_request( @@ -242,7 +256,7 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(None); + let middleware = S3Middleware::new(S3Config::FromAWS); let presigned = middleware .generate_presigned_s3_request( @@ -273,7 +287,7 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(None); + let middleware = S3Middleware::new(S3Config::FromAWS); let presigned = middleware .generate_presigned_s3_request( diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 110a59285..019bcde58 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,7 +1,8 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use rattler_networking::{ - s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware, + s3_middleware::{CustomS3Config, S3Config}, + AuthenticationMiddleware, AuthenticationStorage, S3Middleware, }; use reqwest::Client; use rstest::*; @@ -121,12 +122,12 @@ async fn test_minio_download_repodata( ) { // TODO: also test with AWS environment variables let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); - let middleware = S3Middleware::new(Some(S3Config { + let middleware = S3Middleware::new(S3Config::Custom { auth_storage: auth_storage.clone(), endpoint_url: Url::parse("http://localhost:9000").unwrap(), region: "eu-central-1".into(), force_path_style: true, - })); + }); let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) From 1b1fb87a9cc0c26e7e4baa2db7ba7173a957e151 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 18:26:18 +0100 Subject: [PATCH 25/77] fix --- crates/rattler_networking/tests/integration_test.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 019bcde58..689baca9a 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,8 +1,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use rattler_networking::{ - s3_middleware::{CustomS3Config, S3Config}, - AuthenticationMiddleware, AuthenticationStorage, S3Middleware, + s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware, }; use reqwest::Client; use rstest::*; From 8cb2cf8149839a89ef5d206b0e9d05d691e23091 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 21:25:03 +0100 Subject: [PATCH 26/77] wip --- crates/rattler-bin/src/commands/create.rs | 4 +- crates/rattler/src/cli/auth.rs | 38 +++- .../rattler_networking/src/s3_middleware.rs | 189 ++++++++++-------- .../tests/integration_test.rs | 3 +- 4 files changed, 148 insertions(+), 86 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 570a7b110..d941b0d31 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -21,7 +21,7 @@ use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform, PrefixRecord, RepoDataRecord, Version, }; -use rattler_networking::{AuthenticationMiddleware, AuthenticationStorage}; +use rattler_networking::{s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage}; use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig}; use rattler_solve::{ libsolv_c::{self}, @@ -153,7 +153,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { authentication_storage, ))) .with(rattler_networking::OciMiddleware) - .with(rattler_networking::S3Middleware::new(None)) + .with(rattler_networking::S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::default())) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 0873e8741..d214587e1 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -24,6 +24,18 @@ pub struct LoginArgs { /// The token to use on anaconda.org / quetz authentication #[clap(long)] conda_token: Option, + + /// The AWS access key ID + #[clap(long)] + aws_access_key_id: Option, + + /// The AWS secret access key + #[clap(long)] + aws_secret_access_key: Option, + + /// The AWS session token + #[clap(long)] + aws_session_token: Option, } #[derive(Parser, Debug)] @@ -71,6 +83,14 @@ pub enum AuthenticationCLIError { #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")] AnacondaOrgBadMethod, + /// AWS authentication needs an access key ID and a secret access key. + #[error("Secret access key must be provided when using AWS authentication")] + MissingSecretAccessKey, + + /// Bad authentication method when using S3 + #[error("Authentication with S3 requires a AWS access key ID and a secret access key. Use `--aws-access-key-id` and `--aws-secret-access-key` to provide them")] + S3BadMethod, + /// Wrapper for errors that are generated from the underlying storage system /// (keyring or file system) #[error("Failed to interact with the authentication storage system")] @@ -79,7 +99,7 @@ pub enum AuthenticationCLIError { fn get_url(url: &str) -> Result { // parse as url and extract host without scheme or port - let host = if url.contains("://") { + let host = if url.contains("http://") || url.contains("https://") { url::Url::parse(url)?.host_str().unwrap().to_string() } else { url.to_string() @@ -110,6 +130,14 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } } else if let Some(token) = args.token { Authentication::BearerToken(token) + } else if let Some(access_key_id) = args.aws_access_key_id { + if args.aws_secret_access_key.is_none() { + return Err(AuthenticationCLIError::MissingSecretAccessKey); + } else { + let secret_access_key = args.aws_secret_access_key.unwrap(); + let session_token = args.aws_session_token; + Authentication::S3Credentials { access_key_id, secret_access_key, session_token } + } } else { return Err(AuthenticationCLIError::NoAuthenticationMethod); }; @@ -122,6 +150,14 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti return Err(AuthenticationCLIError::AnacondaOrgBadMethod); } + if host.starts_with("s3://") && !matches!(auth, Authentication::S3Credentials{..}) { + return Err(AuthenticationCLIError::S3BadMethod); + } + + if matches!(auth, Authentication::S3Credentials{..}) && !host.starts_with("s3://") { + return Err(AuthenticationCLIError::S3BadMethod); + } + storage .store(&host, &auth) .map_err(AuthenticationCLIError::StorageError)?; diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 70bbc817d..c74212cc5 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -1,22 +1,23 @@ //! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket +use core::panic; + use async_trait::async_trait; use aws_config::BehaviorVersion; -use aws_sdk_s3::presigning::{PresignedRequest, PresigningConfig}; +use aws_sdk_s3::presigning::PresigningConfig; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use tracing::debug; use url::Url; use crate::{Authentication, AuthenticationStorage}; /// Configuration for the S3 middleware. -#[derive(Clone)] +#[derive(Clone, Debug)] pub enum S3Config { /// Use the default AWS configuration. FromAWS, /// Use a custom configuration. Custom { - /// The authentication storage to use for the S3 client. - auth_storage: AuthenticationStorage, /// The endpoint URL to use for the S3 client. endpoint_url: Url, /// The region to use for the S3 client. @@ -27,74 +28,94 @@ pub enum S3Config { } /// S3 middleware to authenticate requests. -pub struct S3Middleware { +pub struct S3 { + auth_storage: AuthenticationStorage, config: S3Config, expiration: std::time::Duration, } -/// Create an S3 client for given channel with provided configuration. -pub async fn create_s3_client(config: S3Config, url: Option) -> aws_sdk_s3::Client { - let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; - if let ( - S3Config::Custom { - endpoint_url, - region, - force_path_style, +/// S3 middleware to authenticate requests. +pub struct S3Middleware { + s3: S3, +} + +impl S3Middleware { + /// Create a new S3 middleware. + pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { + Self { + s3: S3 { + config, auth_storage, - }, - Some(url), - ) = (config, url) - { - let auth = auth_storage.get_by_url(url).unwrap(); // todo - let config_builder = match auth { - ( - _, - Some(Authentication::S3Credentials { - access_key_id, - secret_access_key, - session_token, - }), - ) => aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(endpoint_url) - .region(aws_sdk_s3::config::Region::new(region)) - .force_path_style(force_path_style) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - access_key_id, - secret_access_key, - session_token, - None, - "pixi", - )), - (_, Some(_)) => { - panic!("Unsupported authentication method"); // todo proper error message - } - (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(endpoint_url) - .region(aws_sdk_s3::config::Region::new(region)) - .force_path_style(force_path_style), - }; - let aws_config = config_builder.build(); - aws_sdk_s3::Client::from_conf(aws_config) - } else { - // TODO: infer path style from endpoint URL or other means and set - // .force_path_style(true) on builder if necessary - let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); - aws_sdk_s3::Client::from_conf(config) + expiration: std::time::Duration::from_secs(300),} + } } } -impl S3Middleware { +impl S3 { /// Create a new S3 middleware. - pub fn new(config: S3Config) -> Self { + pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { Self { config, + auth_storage, expiration: std::time::Duration::from_secs(300), } } + /// Create an S3 client for given channel with provided configuration. + pub async fn create_s3_client(&self, url: Option) -> aws_sdk_s3::Client { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + if let ( + S3Config::Custom { + endpoint_url, + region, + force_path_style, + }, + Some(url), + ) = (self.config.clone(), url) + { + let auth = self.auth_storage.get_by_url(url).unwrap(); // todo + let config_builder = match auth { + ( + _, + Some(Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + }), + ) => aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "pixi", + )), + (_, Some(_)) => { + panic!("Unsupported authentication method"); // todo proper error message + } + (_, None) => todo!("should use no credentials provider and not sign") + // (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) + // .endpoint_url(endpoint_url) + // .region(aws_sdk_s3::config::Region::new(region)) + // .force_path_style(force_path_style), + }; + let aws_config = config_builder.build(); + aws_sdk_s3::Client::from_conf(aws_config) + } else { + // TODO: infer path style from endpoint URL or other means and set + // .force_path_style(true) on builder if necessary + let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); + aws_sdk_s3::Client::from_conf(config) + } + } + + /// Generate a presigned S3 `GetObject` request. - async fn generate_presigned_s3_request(&self, url: Url) -> MiddlewareResult { - let client = create_s3_client(self.config.clone(), Some(url.clone())).await; + async fn generate_presigned_s3_url(&self, url: Url) -> MiddlewareResult { + let client = self.create_s3_client(Some(url.clone())).await; let bucket_name = url.host_str().expect("Host should be present in S3 URL"); let key = url.path().strip_prefix("/").ok_or_else(|| { @@ -105,13 +126,17 @@ impl S3Middleware { })?; let builder = client.get_object().bucket(bucket_name).key(key); - builder + // if client has no credentials provider, don't presign but use default url + // TODO: implement this + + Url::parse(builder .presigned( PresigningConfig::expires_in(self.expiration) .map_err(reqwest_middleware::Error::middleware)?, ) .await - .map_err(reqwest_middleware::Error::middleware) + .map_err(reqwest_middleware::Error::middleware)? + .uri()).map_err(reqwest_middleware::Error::middleware) } } @@ -126,10 +151,10 @@ impl Middleware for S3Middleware { ) -> MiddlewareResult { if req.url().scheme() == "s3" { let url = req.url().clone(); - let presigned_request = self.generate_presigned_s3_request(url).await?; + let presigned_url = self.s3.generate_presigned_s3_url(url).await?; - *req.url_mut() = Url::parse(presigned_request.uri()) - .map_err(reqwest_middleware::Error::middleware)?; + debug!("Presigned S3 url: {:?}", presigned_url); + *req.url_mut() = presigned_url.clone(); } next.run(req, extensions).await } @@ -157,6 +182,8 @@ mod tests { } } + // TODO: test no auth + #[tokio::test] #[serial] async fn test_presigned_s3_request_endpoint_url() { @@ -169,21 +196,21 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(S3Config::FromAWS); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - let presigned = middleware - .generate_presigned_s3_request( + let presigned = s3 + .generate_presigned_s3_url( Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") .unwrap(), ) .await .unwrap(); assert!( - presigned.uri().to_string().starts_with( + presigned.to_string().starts_with( "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" ), "Unexpected presigned URL: {:?}", - presigned.uri() + presigned ); }) }, @@ -202,20 +229,20 @@ mod tests { ]), move || { Box::pin(async { - let middleware = S3Middleware::new(S3Config::FromAWS); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - let presigned = middleware - .generate_presigned_s3_request( + let presigned = s3 + .generate_presigned_s3_url( Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap() ) .await .unwrap(); assert!( - presigned.uri().to_string().starts_with( + presigned.to_string().starts_with( "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" ), "Unexpected presigned URL: {:?}", - presigned.uri() + presigned ); }) }, @@ -256,19 +283,19 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(S3Config::FromAWS); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - let presigned = middleware - .generate_presigned_s3_request( + let presigned = s3 + .generate_presigned_s3_url( Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") .unwrap(), ) .await .unwrap(); assert!( - presigned.uri().to_string().contains("localhost:8000"), + presigned.to_string().contains("localhost:8000"), "Unexpected presigned URL: {:?}", - presigned.uri() + presigned ); }) }, @@ -287,19 +314,19 @@ region = eu-central-1 ]), move || { Box::pin(async move { - let middleware = S3Middleware::new(S3Config::FromAWS); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - let presigned = middleware - .generate_presigned_s3_request( + let presigned = s3 + .generate_presigned_s3_url( Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") .unwrap(), ) .await .unwrap(); assert!( - presigned.uri().to_string().contains("localhost:9000"), + presigned.to_string().contains("localhost:9000"), "Unexpected presigned URL: {:?}", - presigned.uri() + presigned ); }) }, diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 689baca9a..1cb1b3bcc 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -122,11 +122,10 @@ async fn test_minio_download_repodata( // TODO: also test with AWS environment variables let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); let middleware = S3Middleware::new(S3Config::Custom { - auth_storage: auth_storage.clone(), endpoint_url: Url::parse("http://localhost:9000").unwrap(), region: "eu-central-1".into(), force_path_style: true, - }); + }, auth_storage.clone()); let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) From d8b2cc8e1dee6144051752e2ced8226afd45501d Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 3 Jan 2025 22:59:11 +0100 Subject: [PATCH 27/77] . --- crates/rattler_networking/src/s3_middleware.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index c74212cc5..7ad2f6754 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -28,6 +28,7 @@ pub enum S3Config { } /// S3 middleware to authenticate requests. +#[derive(Clone, Debug)] pub struct S3 { auth_storage: AuthenticationStorage, config: S3Config, @@ -35,6 +36,7 @@ pub struct S3 { } /// S3 middleware to authenticate requests. +#[derive(Clone, Debug)] pub struct S3Middleware { s3: S3, } From 356399419509edd3ded89a70f48a6017276474d1 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sat, 4 Jan 2025 00:12:23 +0100 Subject: [PATCH 28/77] fix python --- crates/rattler-bin/src/commands/create.rs | 9 ++- crates/rattler/src/cli/auth.rs | 10 ++- .../rattler_networking/src/s3_middleware.rs | 27 ++++---- .../tests/integration_test.rs | 13 ++-- py-rattler/rattler/networking/middleware.py | 33 ++++++---- py-rattler/src/lib.rs | 4 +- py-rattler/src/networking/middleware.rs | 66 +++++++++++-------- 7 files changed, 98 insertions(+), 64 deletions(-) diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index d941b0d31..fdf4d03ed 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -21,7 +21,9 @@ use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform, PrefixRecord, RepoDataRecord, Version, }; -use rattler_networking::{s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage}; +use rattler_networking::{ + s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, +}; use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig}; use rattler_solve::{ libsolv_c::{self}, @@ -153,7 +155,10 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { authentication_storage, ))) .with(rattler_networking::OciMiddleware) - .with(rattler_networking::S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::default())) + .with(rattler_networking::S3Middleware::new( + S3Config::FromAWS, + AuthenticationStorage::default(), + )) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index d214587e1..c0f4f6e56 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -136,7 +136,11 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } else { let secret_access_key = args.aws_secret_access_key.unwrap(); let session_token = args.aws_session_token; - Authentication::S3Credentials { access_key_id, secret_access_key, session_token } + Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + } } } else { return Err(AuthenticationCLIError::NoAuthenticationMethod); @@ -150,11 +154,11 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti return Err(AuthenticationCLIError::AnacondaOrgBadMethod); } - if host.starts_with("s3://") && !matches!(auth, Authentication::S3Credentials{..}) { + if host.starts_with("s3://") && !matches!(auth, Authentication::S3Credentials { .. }) { return Err(AuthenticationCLIError::S3BadMethod); } - if matches!(auth, Authentication::S3Credentials{..}) && !host.starts_with("s3://") { + if matches!(auth, Authentication::S3Credentials { .. }) && !host.starts_with("s3://") { return Err(AuthenticationCLIError::S3BadMethod); } diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 7ad2f6754..b8d708f96 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -47,8 +47,9 @@ impl S3Middleware { Self { s3: S3 { config, - auth_storage, - expiration: std::time::Duration::from_secs(300),} + auth_storage, + expiration: std::time::Duration::from_secs(300), + }, } } } @@ -98,7 +99,7 @@ impl S3 { (_, Some(_)) => { panic!("Unsupported authentication method"); // todo proper error message } - (_, None) => todo!("should use no credentials provider and not sign") + (_, None) => todo!("should use no credentials provider and not sign"), // (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) // .endpoint_url(endpoint_url) // .region(aws_sdk_s3::config::Region::new(region)) @@ -114,7 +115,6 @@ impl S3 { } } - /// Generate a presigned S3 `GetObject` request. async fn generate_presigned_s3_url(&self, url: Url) -> MiddlewareResult { let client = self.create_s3_client(Some(url.clone())).await; @@ -131,14 +131,17 @@ impl S3 { // if client has no credentials provider, don't presign but use default url // TODO: implement this - Url::parse(builder - .presigned( - PresigningConfig::expires_in(self.expiration) - .map_err(reqwest_middleware::Error::middleware)?, - ) - .await - .map_err(reqwest_middleware::Error::middleware)? - .uri()).map_err(reqwest_middleware::Error::middleware) + Url::parse( + builder + .presigned( + PresigningConfig::expires_in(self.expiration) + .map_err(reqwest_middleware::Error::middleware)?, + ) + .await + .map_err(reqwest_middleware::Error::middleware)? + .uri(), + ) + .map_err(reqwest_middleware::Error::middleware) } } diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 1cb1b3bcc..8a193ffa2 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -121,11 +121,14 @@ async fn test_minio_download_repodata( ) { // TODO: also test with AWS environment variables let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); - let middleware = S3Middleware::new(S3Config::Custom { - endpoint_url: Url::parse("http://localhost:9000").unwrap(), - region: "eu-central-1".into(), - force_path_style: true, - }, auth_storage.clone()); + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index c6800ff95..49197ea5f 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -1,5 +1,4 @@ from __future__ import annotations -from pathlib import Path from rattler.rattler import ( PyAuthenticationMiddleware, @@ -141,18 +140,29 @@ class S3Config: >>> middleware = S3Middleware(config) >>> middleware S3Middleware() + >>> S3Config() + S3Config(aws sdk) >>> ``` """ - def __init__(self, endpoint_url: str, region: str, force_path_style: bool) -> None: + def __init__( + self, endpoint_url: str | None = None, region: str | None = None, force_path_style: bool | None = None + ) -> None: self._config = PyS3Config(endpoint_url, region, force_path_style) - self.endpoint_url = endpoint_url - self.region = region - self.force_path_style = force_path_style + if (endpoint_url is None) != (region is None) or (endpoint_url is None) != (force_path_style is None): + raise ValueError("Invalid arguments for S3Config") + self._endpoint_url = endpoint_url + self._region = region + self._force_path_style = force_path_style def __repr__(self) -> str: - return f"{type(self).__name__}({self.endpoint_url}, {self.region}, {self.force_path_style})" + inner = ( + f"{self._endpoint_url}, {self._region}, {self._force_path_style}" + if self._endpoint_url is not None + else "aws sdk" + ) + return f"{type(self).__name__}({inner})" class S3Middleware: @@ -172,13 +182,10 @@ class S3Middleware: ``` """ - def __init__( - self, config: S3Config | None = None - ) -> None: - if config is not None: - self._middleware = PyS3Middleware(config._config) - else: - self._middleware = PyS3Middleware() + def __init__(self, config: S3Config | None = None) -> None: + if config is None: + config = S3Config() + self._middleware = PyS3Middleware(config._config) def __repr__(self) -> str: return f"{type(self).__name__}()" diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index 90b3daad9..c88ef17b4 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -52,8 +52,8 @@ use match_spec::PyMatchSpec; use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ - PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, - PyS3Middleware, PyS3Config, + PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, PyS3Config, + PyS3Middleware, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 0c29de901..ca8cf8652 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -1,6 +1,7 @@ use pyo3::{pyclass, pymethods, FromPyObject, PyResult}; use rattler_networking::{ - mirror_middleware::Mirror, s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, OciMiddleware, S3Middleware + mirror_middleware::Mirror, s3_middleware::S3Config, AuthenticationMiddleware, + AuthenticationStorage, GCSMiddleware, MirrorMiddleware, OciMiddleware, S3Middleware, }; use std::collections::HashMap; use url::Url; @@ -114,37 +115,54 @@ impl From for GCSMiddleware { } } -#[pyclass] #[derive(Clone)] +#[pyclass] pub struct PyS3Config { - pub(crate) endpoint_url: Url, - pub(crate) region: String, - pub(crate) force_path_style: bool, + // non-trivial enums are not supported by pyo3 as pyclasses + custom: Option, +} + +#[derive(Clone)] +struct PyS3ConfigCustom { + endpoint_url: Url, + region: String, + force_path_style: bool, } #[pymethods] impl PyS3Config { #[new] + #[pyo3(signature = (endpoint_url=None, region=None, force_path_style=None))] pub fn __init__( - endpoint_url: String, - region: String, - force_path_style: bool, + endpoint_url: Option, + region: Option, + force_path_style: Option, ) -> PyResult { - Ok(Self { - endpoint_url: Url::parse(&endpoint_url).map_err(PyRattlerError::from)?, - region, - force_path_style, - }) + match (endpoint_url, region, force_path_style) { + (Some(endpoint_url), Some(region), Some(force_path_style)) => Ok(Self { + custom: Some(PyS3ConfigCustom { + endpoint_url: Url::parse(&endpoint_url) + .map_err(PyRattlerError::from) + .unwrap(), + region, + force_path_style, + }), + }), + (None, None, None) => Ok(Self { custom: None }), + _ => unreachable!("Case handled in python"), + } } } impl From for S3Config { fn from(_value: PyS3Config) -> Self { - S3Config { - auth_storage: AuthenticationStorage::default(), - endpoint_url: _value.endpoint_url, - region: _value.region, - force_path_style: _value.force_path_style, + match _value.custom { + None => S3Config::FromAWS, + Some(custom) => S3Config::Custom { + endpoint_url: custom.endpoint_url, + region: custom.region, + force_path_style: custom.force_path_style, + }, } } } @@ -152,25 +170,19 @@ impl From for S3Config { #[pyclass] #[derive(Clone)] pub struct PyS3Middleware { - pub(crate) s3_config: Option, + pub(crate) s3_config: PyS3Config, } #[pymethods] impl PyS3Middleware { #[new] - #[pyo3(signature = (s3_config=None))] - pub fn __init__( - s3_config: Option, - ) -> PyResult { + pub fn __init__(s3_config: PyS3Config) -> PyResult { Ok(Self { s3_config }) } } impl From for S3Middleware { fn from(_value: PyS3Middleware) -> Self { - match _value.s3_config { - Some(config) => S3Middleware::new(Some(config.into())), - None => S3Middleware::new(None), - } + S3Middleware::new(_value.s3_config.into(), AuthenticationStorage::default()) } } From 259a744030f78930c7b191253735075f3dc42ee2 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sat, 4 Jan 2025 00:38:41 +0100 Subject: [PATCH 29/77] fix --- .github/workflows/rust-compile.yml | 4 +++- crates/rattler_networking/src/s3_middleware.rs | 11 +++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index b32501825..d3ce3e3e5 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -74,7 +74,7 @@ jobs: - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } + - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest } - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } @@ -143,6 +143,8 @@ jobs: uses: prefix-dev/setup-pixi@v0.8.1 with: run-install: false + # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file + post-cleanup: false - name: Install minio if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index b8d708f96..6571d50d4 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -6,7 +6,6 @@ use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::PresigningConfig; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; -use tracing::debug; use url::Url; use crate::{Authentication, AuthenticationStorage}; @@ -45,11 +44,7 @@ impl S3Middleware { /// Create a new S3 middleware. pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { Self { - s3: S3 { - config, - auth_storage, - expiration: std::time::Duration::from_secs(300), - }, + s3: S3::new(config, auth_storage) } } } @@ -157,8 +152,6 @@ impl Middleware for S3Middleware { if req.url().scheme() == "s3" { let url = req.url().clone(); let presigned_url = self.s3.generate_presigned_s3_url(url).await?; - - debug!("Presigned S3 url: {:?}", presigned_url); *req.url_mut() = presigned_url.clone(); } next.run(req, extensions).await @@ -189,6 +182,8 @@ mod tests { // TODO: test no auth + // TODO: test S3Config::Custom + #[tokio::test] #[serial] async fn test_presigned_s3_request_endpoint_url() { From d7bd54e206fd2aca11964cb6cf6504c5b94aacbd Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sat, 4 Jan 2025 00:42:31 +0100 Subject: [PATCH 30/77] fmt --- crates/rattler_networking/src/s3_middleware.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 6571d50d4..98dbafe96 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -44,7 +44,7 @@ impl S3Middleware { /// Create a new S3 middleware. pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { Self { - s3: S3::new(config, auth_storage) + s3: S3::new(config, auth_storage), } } } From 2781798fbc028629093c14e35121b887b21c08e6 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 5 Jan 2025 14:53:29 +0100 Subject: [PATCH 31/77] Add support for public buckets and fix tests --- crates/rattler/src/cli/auth.rs | 26 +-- .../rattler_networking/src/s3_middleware.rs | 169 +++++++++++++----- .../tests/integration_test.rs | 78 +++++++- 3 files changed, 213 insertions(+), 60 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index c0f4f6e56..7e6e79e42 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -25,17 +25,17 @@ pub struct LoginArgs { #[clap(long)] conda_token: Option, - /// The AWS access key ID + /// The S3 access key ID #[clap(long)] - aws_access_key_id: Option, + s3_access_key_id: Option, - /// The AWS secret access key + /// The S3 secret access key #[clap(long)] - aws_secret_access_key: Option, + s3_secret_access_key: Option, - /// The AWS session token + /// The S3 session token #[clap(long)] - aws_session_token: Option, + s3_session_token: Option, } #[derive(Parser, Debug)] @@ -83,12 +83,12 @@ pub enum AuthenticationCLIError { #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")] AnacondaOrgBadMethod, - /// AWS authentication needs an access key ID and a secret access key. - #[error("Secret access key must be provided when using AWS authentication")] + /// S3 authentication needs an access key ID and a secret access key. + #[error("Secret access key must be provided when using S3 authentication")] MissingSecretAccessKey, /// Bad authentication method when using S3 - #[error("Authentication with S3 requires a AWS access key ID and a secret access key. Use `--aws-access-key-id` and `--aws-secret-access-key` to provide them")] + #[error("Authentication with S3 requires a S3 access key ID and a secret access key. Use `--s3-access-key-id` and `--s3-secret-access-key` to provide them")] S3BadMethod, /// Wrapper for errors that are generated from the underlying storage system @@ -130,12 +130,12 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } } else if let Some(token) = args.token { Authentication::BearerToken(token) - } else if let Some(access_key_id) = args.aws_access_key_id { - if args.aws_secret_access_key.is_none() { + } else if let Some(access_key_id) = args.s3_access_key_id { + if args.s3_secret_access_key.is_none() { return Err(AuthenticationCLIError::MissingSecretAccessKey); } else { - let secret_access_key = args.aws_secret_access_key.unwrap(); - let session_token = args.aws_session_token; + let secret_access_key = args.s3_secret_access_key.unwrap(); + let session_token = args.s3_session_token; Authentication::S3Credentials { access_key_id, secret_access_key, diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 98dbafe96..d01a523fd 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -1,6 +1,5 @@ //! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket -use core::panic; - +use anyhow::Error; use async_trait::async_trait; use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::PresigningConfig; @@ -59,9 +58,13 @@ impl S3 { } } - /// Create an S3 client for given channel with provided configuration. - pub async fn create_s3_client(&self, url: Option) -> aws_sdk_s3::Client { - let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + /// Create an S3 client. + /// + /// # Arguments + /// + /// * `url` - The S3 URL to obtain authentication information from the authentication storage. + /// Only respected for custom (non-AWS-based) configuration. + pub async fn create_s3_client(&self, url: Option) -> Result { if let ( S3Config::Custom { endpoint_url, @@ -80,41 +83,62 @@ impl S3 { secret_access_key, session_token, }), - ) => aws_sdk_s3::config::Builder::from(&sdk_config) - .endpoint_url(endpoint_url) - .region(aws_sdk_s3::config::Region::new(region)) - .force_path_style(force_path_style) - .credentials_provider(aws_sdk_s3::config::Credentials::new( - access_key_id, - secret_access_key, - session_token, - None, - "pixi", - )), + ) => { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "pixi", + )) + } (_, Some(_)) => { - panic!("Unsupported authentication method"); // todo proper error message + return Err(anyhow::anyhow!("Unsupported authentication method")); + } + (_, None) => { + eprintln!("No authentication found"); + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .no_credentials() // Turn off request signing + .load() + .await; + aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) } - (_, None) => todo!("should use no credentials provider and not sign"), - // (_, None) => aws_sdk_s3::config::Builder::from(&sdk_config) - // .endpoint_url(endpoint_url) - // .region(aws_sdk_s3::config::Region::new(region)) - // .force_path_style(force_path_style), }; - let aws_config = config_builder.build(); - aws_sdk_s3::Client::from_conf(aws_config) + let s3_config = config_builder.build(); + Ok(aws_sdk_s3::Client::from_conf(s3_config)) } else { - // TODO: infer path style from endpoint URL or other means and set - // .force_path_style(true) on builder if necessary - let config = aws_sdk_s3::config::Builder::from(&sdk_config).build(); - aws_sdk_s3::Client::from_conf(config) + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + // Infer if we expect path-style addressing from the endpoint URL. + if let Some(endpoint_url) = sdk_config.endpoint_url() { + // If the endpoint URL is localhost, we most likely have to use path-style addressing. + if endpoint_url.starts_with("http://localhost") { + s3_config_builder = s3_config_builder.force_path_style(true); + } + } + let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); + Ok(client) } } /// Generate a presigned S3 `GetObject` request. async fn generate_presigned_s3_url(&self, url: Url) -> MiddlewareResult { - let client = self.create_s3_client(Some(url.clone())).await; + let client = self.create_s3_client(Some(url.clone())).await?; - let bucket_name = url.host_str().expect("Host should be present in S3 URL"); + let bucket_name = url.host_str().ok_or_else(|| { + reqwest_middleware::Error::middleware(std::io::Error::new( + std::io::ErrorKind::Other, + "Host should be present in S3 URL", + )) + })?; let key = url.path().strip_prefix("/").ok_or_else(|| { reqwest_middleware::Error::middleware(std::io::Error::new( std::io::ErrorKind::Other, @@ -123,8 +147,6 @@ impl S3 { })?; let builder = client.get_object().bucket(bucket_name).key(key); - // if client has no credentials provider, don't presign but use default url - // TODO: implement this Url::parse( builder @@ -180,10 +202,6 @@ mod tests { } } - // TODO: test no auth - - // TODO: test S3Config::Custom - #[tokio::test] #[serial] async fn test_presigned_s3_request_endpoint_url() { @@ -209,8 +227,7 @@ mod tests { presigned.to_string().starts_with( "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" ), - "Unexpected presigned URL: {:?}", - presigned + "Unexpected presigned URL: {presigned:?}" ); }) }, @@ -241,8 +258,7 @@ mod tests { presigned.to_string().starts_with( "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" ), - "Unexpected presigned URL: {:?}", - presigned + "Unexpected presigned URL: {presigned:?}" ); }) }, @@ -294,8 +310,7 @@ region = eu-central-1 .unwrap(); assert!( presigned.to_string().contains("localhost:8000"), - "Unexpected presigned URL: {:?}", - presigned + "Unexpected presigned URL: {presigned:?}" ); }) }, @@ -325,12 +340,78 @@ region = eu-central-1 .unwrap(); assert!( presigned.to_string().contains("localhost:9000"), - "Unexpected presigned URL: {:?}", - presigned + "Unexpected presigned URL: {presigned:?}" ); }) }, ) .await; } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_custom_config() { + let temp_dir = tempdir().unwrap(); + let aws_config = r#" + { + "s3://rattler-s3-testing/my-channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } + } + "#; + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, aws_config).unwrap(); + let s3 = S3::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(), + ); + + let presigned = s3 + .generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap(); + assert_eq!( + presigned.path(), + "/rattler-s3-testing/my-channel/noarch/repodata.json" + ); + assert_eq!(presigned.scheme(), "http"); + assert_eq!(presigned.host_str().unwrap(), "localhost"); + assert!(presigned.query().unwrap().contains("X-Amz-Credential")); + } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_public_bucket() { + let s3 = S3::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + AuthenticationStorage::new(), // empty auth storage + ); + + let presigned = s3 + .generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap(); + assert!( + presigned.to_string() + == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", + "Unexpected presigned URL: {presigned:?}" + ); + } } diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 8a193ffa2..2035f783c 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -9,7 +9,7 @@ use serial_test::serial; use tempfile::{tempdir, TempDir}; use url::Url; -/* ----------------------------------- MINIO UTILS ---------------------------------- */ +/* -------------------------------------- UTILS ------------------------------------- */ struct MinioServer { handle: std::process::Child, @@ -42,7 +42,10 @@ impl MinioServer { impl Drop for MinioServer { fn drop(&mut self) { eprintln!("Shutting down Minio server (PID={})", self.handle.id()); - self.handle.kill().expect("Failed to kill Minio server"); + match self.handle.kill() { + Ok(_) => {} + Err(e) => eprintln!("Failed to kill Minio server: {e}"), + } } } @@ -87,6 +90,21 @@ fn init_channel() { ); } +async fn with_env( + env: HashMap<&str, &str>, + f: impl FnOnce() -> std::pin::Pin + Send>>, +) { + for (key, value) in &env { + std::env::set_var(key, value); + } + f().await; + for (key, _) in env { + std::env::remove_var(key); + } +} + +/* ------------------------------------ FIXTURES ------------------------------------ */ + #[fixture] fn minio_server() -> MinioServer { let server = MinioServer::new(); @@ -112,6 +130,23 @@ fn auth_file() -> (TempDir, std::path::PathBuf) { (temp_dir, credentials_path) } +#[fixture] +fn aws_config() -> (TempDir, std::path::PathBuf) { + let temp_dir = tempdir().unwrap(); + let aws_config = r#" +[profile default] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +endpoint_url = http://localhost:9000 +region = eu-central-1 +"#; + let aws_config_path = temp_dir.path().join("aws.config"); + std::fs::write(&aws_config_path, aws_config).unwrap(); + (temp_dir, aws_config_path) +} + +/* -------------------------------------- TESTS ------------------------------------- */ + #[rstest] #[tokio::test] #[serial] @@ -119,7 +154,6 @@ async fn test_minio_download_repodata( #[allow(unused_variables)] minio_server: MinioServer, auth_file: (TempDir, std::path::PathBuf), ) { - // TODO: also test with AWS environment variables let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); let middleware = S3Middleware::new( S3Config::Custom { @@ -146,3 +180,41 @@ async fn test_minio_download_repodata( let body = result.text().await.unwrap(); assert!(body.contains("test-package-0.1-0.tar.bz2")); } + +#[rstest] +#[tokio::test] +#[serial] +async fn test_minio_download_repodata_aws_profile( + #[allow(unused_variables)] minio_server: MinioServer, + aws_config: (TempDir, std::path::PathBuf), +) { + with_env( + HashMap::from([ + ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), + ("AWS_PROFILE", "default"), + ]), + move || { + Box::pin(async move { + let auth_storage = AuthenticationStorage::new(); // empty storage + let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); + }) + }, + ) + .await; +} From 93eb440c5b58fe59cb5a70c39dac2a8e3eff8f6f Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 5 Jan 2025 15:17:04 +0100 Subject: [PATCH 32/77] Apply suggestions from code review Co-authored-by: Pavel Zwerschke --- crates/rattler_networking/Cargo.toml | 2 +- crates/rattler_networking/src/authentication_storage/storage.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 1798ccf3b..5ee094053 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls", "s3"] +default = ["native-tls"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 1f8176207..b9d7503b8 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -176,7 +176,7 @@ impl AuthenticationStorage { return Ok((url, None)); } } - _ => return Ok((url, None)), // No more subdomains to check + _ => return Ok((url, None)), // No more subpaths to check } } Ok(Some(credentials)) => return Ok((url, Some(credentials))), From 521beb1785be065b938e4f2cdf0447d22923d1dd Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 5 Jan 2025 18:04:57 +0100 Subject: [PATCH 33/77] Add xref for localhost heuristic --- .../src/authentication_storage/storage.rs | 2 +- crates/rattler_networking/src/s3_middleware.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index b9d7503b8..6d09ce1b0 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -176,7 +176,7 @@ impl AuthenticationStorage { return Ok((url, None)); } } - _ => return Ok((url, None)), // No more subpaths to check + _ => return Ok((url, None)), // No more subpaths to check } } Ok(Some(credentials)) => return Ok((url, Some(credentials))), diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index d01a523fd..5ebf3dcbd 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -101,7 +101,7 @@ impl S3 { return Err(anyhow::anyhow!("Unsupported authentication method")); } (_, None) => { - eprintln!("No authentication found"); + tracing::info!("No authentication found, assuming bucket is public"); let sdk_config = aws_config::defaults(BehaviorVersion::latest()) .no_credentials() // Turn off request signing .load() @@ -119,7 +119,10 @@ impl S3 { let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); // Infer if we expect path-style addressing from the endpoint URL. if let Some(endpoint_url) = sdk_config.endpoint_url() { - // If the endpoint URL is localhost, we most likely have to use path-style addressing. + // If the endpoint URL is localhost, we probably have to use path-style addressing. + // There are certainly more edge cases, but this is a valid start to make the + // integration tests with minIO work. + // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 if endpoint_url.starts_with("http://localhost") { s3_config_builder = s3_config_builder.force_path_style(true); } From bba66f456ea327747cf65b7f6bef6637c245ff78 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 5 Jan 2025 18:59:12 +0100 Subject: [PATCH 34/77] Allow FromAWS for public bucket --- .../rattler_networking/src/s3_middleware.rs | 54 +++++++++++++++++-- .../tests/integration_test.rs | 51 ++++++++++++++++++ 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 5ebf3dcbd..e709b5af1 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -115,7 +115,19 @@ impl S3 { let s3_config = config_builder.build(); Ok(aws_sdk_s3::Client::from_conf(s3_config)) } else { - let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + let mut sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + if let Some(credentials_provider) = sdk_config.credentials_provider() { + let creds = credentials_provider.as_ref().provide_credentials().await; + if creds.is_ok() { + tracing::info!("Using AWS credentials from environment via default provider"); + } else { + tracing::warn!("No AWS credentials found, assuming bucket is public"); + sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .no_credentials() + .load() + .await; + } + } let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); // Infer if we expect path-style addressing from the endpoint URL. if let Some(endpoint_url) = sdk_config.endpoint_url() { @@ -281,7 +293,11 @@ region = eu-central-1 [profile packages] aws_access_key_id = minioadmin aws_secret_access_key = minioadmin -endpoint_url = http://localhost:8000 +endpoint_url = http://localhost:9000 +region = eu-central-1 + +[profile public] +endpoint_url = http://localhost:9000 region = eu-central-1 "#; let aws_config_path = temp_dir.path().join("aws.config"); @@ -312,7 +328,7 @@ region = eu-central-1 .await .unwrap(); assert!( - presigned.to_string().contains("localhost:8000"), + presigned.to_string().contains("localhost:9000"), "Unexpected presigned URL: {presigned:?}" ); }) @@ -417,4 +433,36 @@ region = eu-central-1 "Unexpected presigned URL: {presigned:?}" ); } + + #[rstest] + #[tokio::test] + #[serial] + async fn test_presigned_s3_request_public_bucket_aws( + aws_config: (TempDir, std::path::PathBuf), + ) { + with_env( + HashMap::from([ + ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), + ("AWS_PROFILE", "public"), + ]), + move || { + Box::pin(async move { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let presigned = s3 + .generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .unwrap(), + ) + .await + .unwrap(); + assert!( + presigned.to_string() + == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", + "Unexpected presigned URL: {presigned:?}" + ); + }) + }, + ) + .await; + } } diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 2035f783c..f29cda4fb 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -139,6 +139,10 @@ aws_access_key_id = minioadmin aws_secret_access_key = minioadmin endpoint_url = http://localhost:9000 region = eu-central-1 + +[profile public] +endpoint_url = http://localhost:9000 +region = eu-central-1 "#; let aws_config_path = temp_dir.path().join("aws.config"); std::fs::write(&aws_config_path, aws_config).unwrap(); @@ -218,3 +222,50 @@ async fn test_minio_download_repodata_aws_profile( ) .await; } + +#[rstest] +#[tokio::test] +#[serial] +async fn test_minio_download_aws_profile_public( + #[allow(unused_variables)] minio_server: MinioServer, + aws_config: (TempDir, std::path::PathBuf), +) { + // Make bucket public + run_subprocess( + "mc", + &["anonymous", "set", "download", "local/rattler-s3-testing"], + &HashMap::from([( + "MC_HOST_local", + "http://minioadmin:minioadmin@localhost:9000", + )]), + ); + with_env( + HashMap::from([ + ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), + ("AWS_PROFILE", "public"), + ]), + move || { + Box::pin(async move { + let auth_storage = AuthenticationStorage::new(); // empty storage + let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); + }) + }, + ) + .await; +} From cd7cdafc6dad9cf3947aa3fc3bef9c038b3c9d85 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 5 Jan 2025 19:42:36 +0100 Subject: [PATCH 35/77] empty commit From 89758bcfa8e34f275ed557b00ef7038442afc80c Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 5 Jan 2025 20:34:31 +0100 Subject: [PATCH 36/77] fix --- crates/rattler/src/cli/auth.rs | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index 7e6e79e42..f19b528f2 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -83,10 +83,6 @@ pub enum AuthenticationCLIError { #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")] AnacondaOrgBadMethod, - /// S3 authentication needs an access key ID and a secret access key. - #[error("Secret access key must be provided when using S3 authentication")] - MissingSecretAccessKey, - /// Bad authentication method when using S3 #[error("Authentication with S3 requires a S3 access key ID and a secret access key. Use `--s3-access-key-id` and `--s3-secret-access-key` to provide them")] S3BadMethod, @@ -130,17 +126,12 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } } else if let Some(token) = args.token { Authentication::BearerToken(token) - } else if let Some(access_key_id) = args.s3_access_key_id { - if args.s3_secret_access_key.is_none() { - return Err(AuthenticationCLIError::MissingSecretAccessKey); - } else { - let secret_access_key = args.s3_secret_access_key.unwrap(); - let session_token = args.s3_session_token; - Authentication::S3Credentials { - access_key_id, - secret_access_key, - session_token, - } + } else if let (Some(access_key_id), Some(secret_access_key)) = (args.s3_access_key_id, args.s3_secret_access_key) { + let session_token = args.s3_session_token; + Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, } } else { return Err(AuthenticationCLIError::NoAuthenticationMethod); From e1ca81d840724ba438d3ecbc10c213d2b2c1243f Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 5 Jan 2025 20:42:20 +0100 Subject: [PATCH 37/77] fix --- crates/rattler/src/cli/auth.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index f19b528f2..d4d27beef 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -126,7 +126,9 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } } else if let Some(token) = args.token { Authentication::BearerToken(token) - } else if let (Some(access_key_id), Some(secret_access_key)) = (args.s3_access_key_id, args.s3_secret_access_key) { + } else if let (Some(access_key_id), Some(secret_access_key)) = + (args.s3_access_key_id, args.s3_secret_access_key) + { let session_token = args.s3_session_token; Authentication::S3Credentials { access_key_id, From 382474e751c39be10cdd844aca3fdc35677e3115 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 5 Jan 2025 21:22:29 +0100 Subject: [PATCH 38/77] fix tests, add custom + public integration test --- .config/nextest.toml | 6 +++ .../tests/integration_test.rs | 40 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000..76be4a55f --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,6 @@ +[test-groups] +serial-integration = { max-threads = 1 } + +[[profile.default.overrides]] +filter = 'package(rattler_networking) and (test(#test_minio*) | test(s3_middleware::tests))' +test-group = 'serial-integration' diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index f29cda4fb..5191daa39 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -185,6 +185,46 @@ async fn test_minio_download_repodata( assert!(body.contains("test-package-0.1-0.tar.bz2")); } +#[rstest] +#[tokio::test] +#[serial] +async fn test_minio_download_repodata_public(#[allow(unused_variables)] minio_server: MinioServer) { + // Make bucket public + run_subprocess( + "mc", + &["anonymous", "set", "download", "local/rattler-s3-testing"], + &HashMap::from([( + "MC_HOST_local", + "http://minioadmin:minioadmin@localhost:9000", + )]), + ); + let auth_storage = AuthenticationStorage::new(); // empty storage + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); +} + #[rstest] #[tokio::test] #[serial] From b97a0d3f4a6417e8ad08651aeed7c1b84b41a782 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Fri, 10 Jan 2025 18:10:01 +0100 Subject: [PATCH 39/77] Fix some tests --- .github/workflows/rust-compile.yml | 4 + Cargo.toml | 2 +- crates/rattler_networking/Cargo.toml | 3 +- .../rattler_networking/src/s3_middleware.rs | 225 +++++-------- .../tests/integration_test.rs | 310 +++++++----------- 5 files changed, 219 insertions(+), 325 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index d3ce3e3e5..0e7852fbb 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -152,6 +152,10 @@ jobs: pixi global install minio-server pixi global install minio-client + - name: Start minio + if: ${{ !matrix.skip-tests }} + run: pixi exec minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + - name: Run tests if: ${{ !matrix.skip-tests }} env: diff --git a/Cargo.toml b/Cargo.toml index 7c1ee5b6d..e14a3f73a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -146,7 +146,7 @@ sysinfo = "0.33.1" tar = "0.4.43" tempdir = "0.3.7" tempfile = "3.15.0" -temp-env = "0.3.6" +temp-env = { version = "0.3.6", features = ["async_closure"] } test-log = "0.2.16" thiserror = "2.0" tokio = { version = "1.42.0", default-features = false } diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 5ee094053..3bcdc5a0f 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls"] +default = ["native-tls", "s3"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] @@ -62,4 +62,3 @@ sha2 = { workspace = true } temp-env = { workspace = true } rstest = { workspace = true } rand = { workspace = true } -serial_test = { workspace = true } diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index e709b5af1..089c7a75e 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -25,7 +25,7 @@ pub enum S3Config { }, } -/// S3 middleware to authenticate requests. +/// Wrapper around S3 client. #[derive(Clone, Debug)] pub struct S3 { auth_storage: AuthenticationStorage, @@ -49,7 +49,7 @@ impl S3Middleware { } impl S3 { - /// Create a new S3 middleware. + /// Create a new S3 client wrapper. pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { Self { config, @@ -197,88 +197,58 @@ impl Middleware for S3Middleware { #[cfg(test)] mod tests { - use std::collections::HashMap; - use super::*; use rstest::{fixture, rstest}; - use serial_test::serial; + use temp_env::async_with_vars; use tempfile::{tempdir, TempDir}; - async fn with_env( - env: HashMap<&str, &str>, - f: impl FnOnce() -> std::pin::Pin + Send>>, - ) { - for (key, value) in &env { - std::env::set_var(key, value); - } - f().await; - for (key, _) in env { - std::env::remove_var(key); - } - } - #[tokio::test] - #[serial] async fn test_presigned_s3_request_endpoint_url() { - with_env( - HashMap::from([ - ("AWS_ACCESS_KEY_ID", "minioadmin"), - ("AWS_SECRET_ACCESS_KEY", "minioadmin"), - ("AWS_REGION", "eu-central-1"), - ("AWS_ENDPOINT_URL", "http://custom-aws"), - ]), - move || { - Box::pin(async { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - - let presigned = s3 - .generate_presigned_s3_url( - Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .unwrap(), - ) - .await - .unwrap(); - assert!( - presigned.to_string().starts_with( - "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" - ), - "Unexpected presigned URL: {presigned:?}" - ); - }) + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let presigned = async_with_vars( + [ + ("AWS_ACCESS_KEY_ID", Some("minioadmin")), + ("AWS_SECRET_ACCESS_KEY", Some("minioadmin")), + ("AWS_REGION", Some("eu-central-1")), + ("AWS_ENDPOINT_URL", Some("http://custom-aws")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() }, ) .await; + assert!( + presigned.to_string().starts_with( + "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" + ), + "Unexpected presigned URL: {presigned:?}" + ); } #[tokio::test] - #[serial] async fn test_presigned_s3_request_aws() { - with_env( - HashMap::from([ - ("AWS_ACCESS_KEY_ID", "minioadmin"), - ("AWS_SECRET_ACCESS_KEY", "minioadmin"), - ("AWS_REGION", "eu-central-1"), - ]), - move || { - Box::pin(async { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - - let presigned = s3 - .generate_presigned_s3_url( - Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap() - ) - .await - .unwrap(); - assert!( - presigned.to_string().starts_with( - "https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?" - ), - "Unexpected presigned URL: {presigned:?}" - ); - }) + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let presigned = async_with_vars( + [ + ("AWS_ACCESS_KEY_ID", Some("minioadmin")), + ("AWS_SECRET_ACCESS_KEY", Some("minioadmin")), + ("AWS_REGION", Some("eu-central-1")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() }, ) .await; + assert!(presigned.to_string().starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?"), "Unexpected presigned URL: {presigned:?}" + ); } #[fixture] @@ -307,69 +277,55 @@ region = eu-central-1 #[rstest] #[tokio::test] - #[serial] async fn test_presigned_s3_request_custom_config_from_env( aws_config: (TempDir, std::path::PathBuf), ) { - with_env( - HashMap::from([ - ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), - ("AWS_PROFILE", "packages"), - ]), - move || { - Box::pin(async move { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - - let presigned = s3 - .generate_presigned_s3_url( - Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .unwrap(), - ) - .await - .unwrap(); - assert!( - presigned.to_string().contains("localhost:9000"), - "Unexpected presigned URL: {presigned:?}" - ); - }) + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let presigned = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("packages")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() }, ) .await; + assert!( + presigned.to_string().contains("localhost:9000"), + "Unexpected presigned URL: {presigned:?}" + ); } #[rstest] #[tokio::test] - #[serial] async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { - with_env( - HashMap::from([ - ("AWS_ENDPOINT_URL", "http://localhost:9000"), - ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), - ]), - move || { - Box::pin(async move { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); - - let presigned = s3 - .generate_presigned_s3_url( - Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .unwrap(), - ) - .await - .unwrap(); - assert!( - presigned.to_string().contains("localhost:9000"), - "Unexpected presigned URL: {presigned:?}" - ); - }) + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let presigned = async_with_vars( + [ + ("AWS_ENDPOINT_URL", Some("http://localhost:9000")), + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() }, ) .await; + assert!( + presigned.to_string().contains("localhost:9000"), + "Unexpected presigned URL: {presigned:?}" + ); } - #[rstest] #[tokio::test] - #[serial] async fn test_presigned_s3_request_custom_config() { let temp_dir = tempdir().unwrap(); let aws_config = r#" @@ -408,9 +364,7 @@ region = eu-central-1 assert!(presigned.query().unwrap().contains("X-Amz-Credential")); } - #[rstest] #[tokio::test] - #[serial] async fn test_presigned_s3_request_public_bucket() { let s3 = S3::new( S3Config::Custom { @@ -436,33 +390,28 @@ region = eu-central-1 #[rstest] #[tokio::test] - #[serial] async fn test_presigned_s3_request_public_bucket_aws( aws_config: (TempDir, std::path::PathBuf), ) { - with_env( - HashMap::from([ - ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), - ("AWS_PROFILE", "public"), - ]), - move || { - Box::pin(async move { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); - let presigned = s3 - .generate_presigned_s3_url( - Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .unwrap(), - ) - .await - .unwrap(); - assert!( - presigned.to_string() - == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", - "Unexpected presigned URL: {presigned:?}" - ); - }) + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let presigned = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("public")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() }, ) .await; + assert!( + presigned.to_string() + == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", + "Unexpected presigned URL: {presigned:?}" + ); } } diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 5191daa39..acdaf63d4 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -5,50 +5,12 @@ use rattler_networking::{ }; use reqwest::Client; use rstest::*; -use serial_test::serial; +use temp_env::async_with_vars; use tempfile::{tempdir, TempDir}; use url::Url; /* -------------------------------------- UTILS ------------------------------------- */ -struct MinioServer { - handle: std::process::Child, - #[allow(dead_code)] - directory: tempfile::TempDir, -} - -impl MinioServer { - fn new() -> Self { - let directory = tempfile::tempdir().expect("Failed to create temp directory"); - let args = [ - "server", - directory.path().to_str().unwrap(), - "--address", - "127.0.0.1:9000", - ]; - let handle = std::process::Command::new("minio") - .args(args) - .spawn() - .expect("Failed to start Minio server"); - eprintln!( - "Starting Minio server with args (PID={}): {:?}", - handle.id(), - args - ); - MinioServer { handle, directory } - } -} - -impl Drop for MinioServer { - fn drop(&mut self) { - eprintln!("Shutting down Minio server (PID={})", self.handle.id()); - match self.handle.kill() { - Ok(_) => {} - Err(e) => eprintln!("Failed to kill Minio server: {e}"), - } - } -} - fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::process::Output { let mut command = std::process::Command::new(cmd); command.args(args); @@ -60,90 +22,84 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p output } +/* ------------------------------------ FIXTURES ------------------------------------ */ + +#[fixture] +#[once] +fn minio_host() -> String { + format!( + "http://localhost:{}", + option_env!("MINIO_PORT").unwrap_or("9000") + ) +} + +#[fixture] +#[once] fn init_channel() { - let env = &HashMap::from([( - "MC_HOST_local", - "http://minioadmin:minioadmin@localhost:9000", - )]); - run_subprocess("mc", &["mb", "local/rattler-s3-testing"], env); - run_subprocess( - "mc", - &[ - "cp", - PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") - .to_str() - .unwrap(), - "local/rattler-s3-testing/my-channel/noarch/repodata.json", - ], - env, + let host = format!( + "http://minioadmin:minioadmin@localhost:{}", + option_env!("MINIO_PORT").unwrap_or("9000") ); + let env = &HashMap::from([("MC_HOST_local", host.as_str())]); + for bucket in &[ + "local/rattler-s3-testing", + "local/rattler-s3-testing-public", + ] { + run_subprocess("mc", &["mb", bucket], env); + run_subprocess( + "mc", + &[ + "cp", + PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") + .to_str() + .unwrap(), + format!("{}/my-channel/noarch/repodata.json", bucket).as_str(), + ], + env, + ); + run_subprocess( + "mc", + &[ + "cp", + PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") + .to_str() + .unwrap(), + format!("{}/my-channel/noarch/test-package-0.1-0.tar.bz2", bucket).as_str(), + ], + env, + ); + } + // Make bucket public run_subprocess( "mc", &[ - "cp", - PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") - .to_str() - .unwrap(), - "local/rattler-s3-testing/my-channel/noarch/test-package-0.1-0.tar.bz2", + "anonymous", + "set", + "download", + "local/rattler-s3-testing-public", ], env, ); } -async fn with_env( - env: HashMap<&str, &str>, - f: impl FnOnce() -> std::pin::Pin + Send>>, -) { - for (key, value) in &env { - std::env::set_var(key, value); - } - f().await; - for (key, _) in env { - std::env::remove_var(key); - } -} - -/* ------------------------------------ FIXTURES ------------------------------------ */ - -#[fixture] -fn minio_server() -> MinioServer { - let server = MinioServer::new(); - init_channel(); - server -} - #[fixture] -fn auth_file() -> (TempDir, std::path::PathBuf) { +fn aws_config(minio_host: &str) -> (TempDir, std::path::PathBuf) { let temp_dir = tempdir().unwrap(); - let aws_config = r#" -{ - "s3://rattler-s3-testing/my-channel": { - "S3Credentials": { - "access_key_id": "minioadmin", - "secret_access_key": "minioadmin" - } - } -} -"#; - let credentials_path = temp_dir.path().join("credentials.json"); - std::fs::write(&credentials_path, aws_config).unwrap(); - (temp_dir, credentials_path) -} - -#[fixture] -fn aws_config() -> (TempDir, std::path::PathBuf) { - let temp_dir = tempdir().unwrap(); - let aws_config = r#" + let aws_config = format!( + r#" [profile default] aws_access_key_id = minioadmin aws_secret_access_key = minioadmin -endpoint_url = http://localhost:9000 +endpoint_url = {} region = eu-central-1 [profile public] -endpoint_url = http://localhost:9000 +endpoint_url = {} region = eu-central-1 -"#; +"#, + minio_host.as_str(), + minio_host.as_str() + ); let aws_config_path = temp_dir.path().join("aws.config"); std::fs::write(&aws_config_path, aws_config).unwrap(); (temp_dir, aws_config_path) @@ -153,15 +109,26 @@ region = eu-central-1 #[rstest] #[tokio::test] -#[serial] async fn test_minio_download_repodata( - #[allow(unused_variables)] minio_server: MinioServer, - auth_file: (TempDir, std::path::PathBuf), + minio_host: &str, + #[allow(unused_variables)] init_channel: (), ) { - let auth_storage = AuthenticationStorage::from_file(auth_file.1.as_path()).unwrap(); + let temp_dir = tempdir().unwrap(); + let aws_config = r#" +{ + "s3://rattler-s3-testing/my-channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } +}"#; + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, aws_config).unwrap(); + let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); let middleware = S3Middleware::new( S3Config::Custom { - endpoint_url: Url::parse("http://localhost:9000").unwrap(), + endpoint_url: Url::parse(minio_host).unwrap(), region: "eu-central-1".into(), force_path_style: true, }, @@ -187,21 +154,14 @@ async fn test_minio_download_repodata( #[rstest] #[tokio::test] -#[serial] -async fn test_minio_download_repodata_public(#[allow(unused_variables)] minio_server: MinioServer) { - // Make bucket public - run_subprocess( - "mc", - &["anonymous", "set", "download", "local/rattler-s3-testing"], - &HashMap::from([( - "MC_HOST_local", - "http://minioadmin:minioadmin@localhost:9000", - )]), - ); +async fn test_minio_download_repodata_public( + minio_host: &str, + #[allow(unused_variables)] init_channel: (), +) { let auth_storage = AuthenticationStorage::new(); // empty storage let middleware = S3Middleware::new( S3Config::Custom { - endpoint_url: Url::parse("http://localhost:9000").unwrap(), + endpoint_url: Url::parse(minio_host).unwrap(), region: "eu-central-1".into(), force_path_style: true, }, @@ -215,7 +175,7 @@ async fn test_minio_download_repodata_public(#[allow(unused_variables)] minio_se .build(); let result = download_client - .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .get("s3://rattler-s3-testing-public/my-channel/noarch/repodata.json") .send() .await .unwrap(); @@ -227,85 +187,67 @@ async fn test_minio_download_repodata_public(#[allow(unused_variables)] minio_se #[rstest] #[tokio::test] -#[serial] async fn test_minio_download_repodata_aws_profile( - #[allow(unused_variables)] minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), + #[allow(unused_variables)] init_channel: (), ) { - with_env( - HashMap::from([ - ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), - ("AWS_PROFILE", "default"), - ]), - move || { - Box::pin(async move { - let auth_storage = AuthenticationStorage::new(); // empty storage - let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); - - let download_client = Client::builder().no_gzip().build().unwrap(); - let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) - .with(middleware) - .build(); + let auth_storage = AuthenticationStorage::new(); // empty storage + let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); - let result = download_client - .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .send() - .await - .unwrap(); + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); - assert_eq!(result.status(), 200); - let body = result.text().await.unwrap(); - assert!(body.contains("test-package-0.1-0.tar.bz2")); - }) + let result = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("default")), + ], + async { + download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap() }, ) .await; + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); } #[rstest] #[tokio::test] -#[serial] async fn test_minio_download_aws_profile_public( - #[allow(unused_variables)] minio_server: MinioServer, aws_config: (TempDir, std::path::PathBuf), + #[allow(unused_variables)] init_channel: (), ) { - // Make bucket public - run_subprocess( - "mc", - &["anonymous", "set", "download", "local/rattler-s3-testing"], - &HashMap::from([( - "MC_HOST_local", - "http://minioadmin:minioadmin@localhost:9000", - )]), - ); - with_env( - HashMap::from([ - ("AWS_CONFIG_FILE", aws_config.1.to_str().unwrap()), - ("AWS_PROFILE", "public"), - ]), - move || { - Box::pin(async move { - let auth_storage = AuthenticationStorage::new(); // empty storage - let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); - - let download_client = Client::builder().no_gzip().build().unwrap(); - let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) - .with(middleware) - .build(); - - let result = download_client - .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") - .send() - .await - .unwrap(); + let auth_storage = AuthenticationStorage::new(); // empty storage + let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); - assert_eq!(result.status(), 200); - let body = result.text().await.unwrap(); - assert!(body.contains("test-package-0.1-0.tar.bz2")); - }) + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); + let result = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("public")), + ], + async { + download_client + .get("s3://rattler-s3-testing-public/my-channel/noarch/repodata.json") + .send() + .await + .unwrap() }, ) .await; + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); } From d319fcb47ec61c2f35600a052911a6639bc2de0e Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Mon, 6 Jan 2025 00:00:45 +0100 Subject: [PATCH 40/77] add debug statement --- crates/rattler_networking/src/s3_middleware.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 089c7a75e..a4ad63cf5 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -5,6 +5,7 @@ use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::PresigningConfig; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use tracing::debug; use url::Url; use crate::{Authentication, AuthenticationStorage}; @@ -42,6 +43,7 @@ pub struct S3Middleware { impl S3Middleware { /// Create a new S3 middleware. pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { + debug!("Creating S3 middleware using {:?}", config); Self { s3: S3::new(config, auth_storage), } From 95225a2ae4095c31431b5c53f931c91dbd8511f7 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 10 Jan 2025 23:28:27 +0100 Subject: [PATCH 41/77] fix clippy --- crates/rattler_networking/tests/integration_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index acdaf63d4..fa269dc2d 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -97,8 +97,8 @@ region = eu-central-1 endpoint_url = {} region = eu-central-1 "#, - minio_host.as_str(), - minio_host.as_str() + minio_host, + minio_host ); let aws_config_path = temp_dir.path().join("aws.config"); std::fs::write(&aws_config_path, aws_config).unwrap(); From 8537405541ef9d17d2faf98f230920ba4d6ce2d8 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 10 Jan 2025 23:31:03 +0100 Subject: [PATCH 42/77] fmt --- crates/rattler_networking/tests/integration_test.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index fa269dc2d..64aa01f5b 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -17,7 +17,7 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p for (key, value) in env { command.env(key, value); } - let output = command.output().expect("Failed to run command"); + let output = command.output().unwrap(); assert!(output.status.success()); output } @@ -97,8 +97,7 @@ region = eu-central-1 endpoint_url = {} region = eu-central-1 "#, - minio_host, - minio_host + minio_host, minio_host ); let aws_config_path = temp_dir.path().join("aws.config"); std::fs::write(&aws_config_path, aws_config).unwrap(); From a616161683454bc9ed0b64217b9dc44317f8c6b6 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 13:03:43 +0100 Subject: [PATCH 43/77] Properly run minio as background process --- .github/workflows/rust-compile.yml | 15 +++++++++------ .../rattler_networking/tests/integration_test.rs | 13 ++++++------- pixi.toml | 4 ++-- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 0e7852fbb..8de59618e 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -146,15 +146,14 @@ jobs: # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file post-cleanup: false - - name: Install minio + - name: Start minio for integration tests if: ${{ !matrix.skip-tests }} run: | - pixi global install minio-server pixi global install minio-client - - - name: Start minio - if: ${{ !matrix.skip-tests }} - run: pixi exec minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + pixi global install minio-server + pixi run minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + MINIO_PID=$(echo $!) + echo "MINIO_PID=${MINIO_PID}" >> $GITHUB_ENV - name: Run tests if: ${{ !matrix.skip-tests }} @@ -168,6 +167,10 @@ jobs: ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} + - name: Stop minio + if: ${{ always() && !matrix.skip-tests }} + run: kill -9 -$MINIO_PID + - name: Run doctests if: ${{ !matrix.skip-tests }} run: > diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 64aa01f5b..4bfeb0eab 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -45,7 +45,7 @@ fn init_channel() { "local/rattler-s3-testing", "local/rattler-s3-testing-public", ] { - run_subprocess("mc", &["mb", bucket], env); + run_subprocess("mc", &["mb", "--ignore-existing", bucket], env); run_subprocess( "mc", &[ @@ -53,7 +53,7 @@ fn init_channel() { PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") .to_str() .unwrap(), - format!("{}/my-channel/noarch/repodata.json", bucket).as_str(), + format!("{bucket}/my-channel/noarch/repodata.json").as_str(), ], env, ); @@ -64,7 +64,7 @@ fn init_channel() { PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") .to_str() .unwrap(), - format!("{}/my-channel/noarch/test-package-0.1-0.tar.bz2", bucket).as_str(), + format!("{bucket}/my-channel/noarch/test-package-0.1-0.tar.bz2").as_str(), ], env, ); @@ -90,14 +90,13 @@ fn aws_config(minio_host: &str) -> (TempDir, std::path::PathBuf) { [profile default] aws_access_key_id = minioadmin aws_secret_access_key = minioadmin -endpoint_url = {} +endpoint_url = {minio_host} region = eu-central-1 [profile public] -endpoint_url = {} +endpoint_url = {minio_host} region = eu-central-1 -"#, - minio_host, minio_host +"# ); let aws_config_path = temp_dir.path().join("aws.config"); std::fs::write(&aws_config_path, aws_config).unwrap(); diff --git a/pixi.toml b/pixi.toml index 62dc7bc5f..7bbfe3709 100644 --- a/pixi.toml +++ b/pixi.toml @@ -26,8 +26,8 @@ make = "~=4.3" pkg-config = "~=0.29.2" rust = "~=1.81.0" cmake = "~=3.26.4" -minio-server = "*" -minio-client = "*" +minio-server = ">=2024.12.18" +minio-client = ">=2024.11.21" [target.linux-64.dependencies] clang = ">=18.1.8,<19.0" From 21d2bef0a32f31ed5d6eae5b0121fc519447e52e Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 13:06:03 +0100 Subject: [PATCH 44/77] Remove nextest config --- .config/nextest.toml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml deleted file mode 100644 index 76be4a55f..000000000 --- a/.config/nextest.toml +++ /dev/null @@ -1,6 +0,0 @@ -[test-groups] -serial-integration = { max-threads = 1 } - -[[profile.default.overrides]] -filter = 'package(rattler_networking) and (test(#test_minio*) | test(s3_middleware::tests))' -test-group = 'serial-integration' From d90878504fa22ea4703263c343a2c471139fbdd1 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 16:48:44 +0100 Subject: [PATCH 45/77] Remove killing of minio server --- .github/workflows/rust-compile.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 8de59618e..1d995a351 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -152,8 +152,6 @@ jobs: pixi global install minio-client pixi global install minio-server pixi run minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & - MINIO_PID=$(echo $!) - echo "MINIO_PID=${MINIO_PID}" >> $GITHUB_ENV - name: Run tests if: ${{ !matrix.skip-tests }} @@ -167,10 +165,6 @@ jobs: ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} - - name: Stop minio - if: ${{ always() && !matrix.skip-tests }} - run: kill -9 -$MINIO_PID - - name: Run doctests if: ${{ !matrix.skip-tests }} run: > From 0d941e978012402ab2a878570231afcbe634324a Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 12 Jan 2025 16:54:00 +0100 Subject: [PATCH 46/77] empty commit From 9a581e660af46474e59335f301833e36ff9ba209 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 17:18:49 +0100 Subject: [PATCH 47/77] Cleanup Cargo.toml --- Cargo.toml | 13 +++++++++---- crates/rattler_networking/Cargo.toml | 3 +-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e14a3f73a..641f176f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,9 +72,15 @@ getrandom = { version = "0.2.15", default-features = false } glob = "0.3.2" google-cloud-auth = { version = "0.17.2", default-features = false } google-cloud-token = "0.1.2" -aws-config = "1.5.12" -aws-sdk-s3 = "1.67.0" -aws-runtime = "1.5.2" +aws-config = { version = "1.5.13", default-features = false, features = [ + "rt-tokio", + "rustls", +] } +aws-sdk-s3 = { version = "1.68.0", default-features = false, features = [ + "rt-tokio", + "rustls", + "sigv4a", +] } hex = "0.4.3" hex-literal = "0.4.1" http = "1.2" @@ -129,7 +135,6 @@ serde-value = "0.7.0" serde_with = "3.12.0" serde_yaml = "0.9.34" serde-untagged = "0.1.6" -serial_test = "0.4.0" sha2 = "0.10.8" shlex = "1.3.0" similar-asserts = "1.6.0" diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index c220be103..d29893d53 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -15,7 +15,7 @@ default = ["native-tls", "s3"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] -s3 = ["aws-config", "aws-sdk-s3", "aws-runtime"] +s3 = ["aws-config", "aws-sdk-s3"] [dependencies] anyhow = { workspace = true } @@ -28,7 +28,6 @@ google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } aws-config = { workspace = true, optional = true } aws-sdk-s3 = { workspace = true, optional = true } -aws-runtime = { workspace = true, optional = true } http = { workspace = true } itertools = { workspace = true } keyring = { workspace = true, features = [ From e213be08bb6aed2162ae7f59c3792b37b63ba27c Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 17:45:10 +0100 Subject: [PATCH 48/77] Add cli args conditions --- crates/rattler/src/cli/auth.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index d4d27beef..af7316509 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -26,15 +26,15 @@ pub struct LoginArgs { conda_token: Option, /// The S3 access key ID - #[clap(long)] + #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["host", "token","username", "password", "conda_token"])] s3_access_key_id: Option, /// The S3 secret access key - #[clap(long)] + #[clap(long, requires_all = ["s3_access_key_id"])] s3_secret_access_key: Option, /// The S3 session token - #[clap(long)] + #[clap(long, requires_all = ["s3_access_key_id"])] s3_session_token: Option, } From 63049195a7533d55ac5d4627de54ea8628202c53 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 12 Jan 2025 17:47:41 +0100 Subject: [PATCH 49/77] Update crates/rattler/src/cli/auth.rs --- crates/rattler/src/cli/auth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index af7316509..acf3d899c 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -26,7 +26,7 @@ pub struct LoginArgs { conda_token: Option, /// The S3 access key ID - #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["host", "token","username", "password", "conda_token"])] + #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["host", "token", "username", "password", "conda_token"])] s3_access_key_id: Option, /// The S3 secret access key From 56007207d7db0eaa832376a187f4b504110f9c5b Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 18:48:09 +0100 Subject: [PATCH 50/77] Adjust error messages --- crates/rattler_networking/src/s3_middleware.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index a4ad63cf5..66dde00b2 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -100,7 +100,7 @@ impl S3 { )) } (_, Some(_)) => { - return Err(anyhow::anyhow!("Unsupported authentication method")); + return Err(anyhow::anyhow!("unsupported authentication method")); } (_, None) => { tracing::info!("No authentication found, assuming bucket is public"); @@ -153,15 +153,10 @@ impl S3 { let bucket_name = url.host_str().ok_or_else(|| { reqwest_middleware::Error::middleware(std::io::Error::new( std::io::ErrorKind::Other, - "Host should be present in S3 URL", - )) - })?; - let key = url.path().strip_prefix("/").ok_or_else(|| { - reqwest_middleware::Error::middleware(std::io::Error::new( - std::io::ErrorKind::Other, - "Missing prefix", + "host should be present in S3 URL", )) })?; + let key = url.path().strip_prefix("/").unwrap(); let builder = client.get_object().bucket(bucket_name).key(key); From d17bc894957e2743068adfe733325e726e182ead Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 19:17:28 +0100 Subject: [PATCH 51/77] Fix windows test --- .github/workflows/rust-compile.yml | 8 ++++++-- crates/rattler_networking/tests/integration_test.rs | 13 +++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 1d995a351..c6055ddeb 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -146,12 +146,16 @@ jobs: # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file post-cleanup: false - - name: Start minio for integration tests + - name: Install minio for integration tests if: ${{ !matrix.skip-tests }} run: | pixi global install minio-client pixi global install minio-server - pixi run minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + + - name: Start minio server + if: ${{ !matrix.skip-tests }} + run: | + minio${{ startsWith(matrix.os, 'windows') && '.bat' || '' }} server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & - name: Run tests if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 4bfeb0eab..e2c5d6915 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -41,13 +41,18 @@ fn init_channel() { option_env!("MINIO_PORT").unwrap_or("9000") ); let env = &HashMap::from([("MC_HOST_local", host.as_str())]); + let mc_executable = if cfg!(target_os = "windows") { + "mc.bat" + } else { + "mc" + }; for bucket in &[ "local/rattler-s3-testing", "local/rattler-s3-testing-public", ] { - run_subprocess("mc", &["mb", "--ignore-existing", bucket], env); + run_subprocess(mc_executable, &["mb", "--ignore-existing", bucket], env); run_subprocess( - "mc", + mc_executable, &[ "cp", PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") @@ -58,7 +63,7 @@ fn init_channel() { env, ); run_subprocess( - "mc", + mc_executable, &[ "cp", PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") @@ -71,7 +76,7 @@ fn init_channel() { } // Make bucket public run_subprocess( - "mc", + mc_executable, &[ "anonymous", "set", From 87bb481b66406eb7edb7bd7ea314a2e21f8d8f11 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 12 Jan 2025 19:18:26 +0100 Subject: [PATCH 52/77] empty commit From f92386e5c17271f17e6e709a381618f823c903c1 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 20:47:23 +0100 Subject: [PATCH 53/77] Debug windows test --- crates/rattler_networking/tests/integration_test.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index e2c5d6915..564e32fe3 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -18,7 +18,11 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p command.env(key, value); } let output = command.output().unwrap(); - assert!(output.status.success()); + if !output.status.success() { + eprintln!("Command failed: {:?}", command); + eprintln!("Output: {:?}", String::from_utf8_lossy(&output.stdout)); + eprintln!("Error: {:?}", String::from_utf8_lossy(&output.stderr)); + } output } @@ -41,11 +45,7 @@ fn init_channel() { option_env!("MINIO_PORT").unwrap_or("9000") ); let env = &HashMap::from([("MC_HOST_local", host.as_str())]); - let mc_executable = if cfg!(target_os = "windows") { - "mc.bat" - } else { - "mc" - }; + let mc_executable = if cfg!(windows) { "mc.bat" } else { "mc" }; for bucket in &[ "local/rattler-s3-testing", "local/rattler-s3-testing-public", From 642dc8177e99f155b5f475299211cf3223f6cce7 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Sun, 12 Jan 2025 20:53:02 +0100 Subject: [PATCH 54/77] empty commit From b671bdef18e2c957ada90c67146bba482d43349f Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:15:19 +0100 Subject: [PATCH 55/77] More debug output on windows --- crates/rattler_networking/tests/integration_test.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 564e32fe3..646ab2daf 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -17,7 +17,9 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p for (key, value) in env { command.env(key, value); } - let output = command.output().unwrap(); + let output = command + .output() + .expect(format!("Failed to execute command {:?}", command).as_str()); if !output.status.success() { eprintln!("Command failed: {:?}", command); eprintln!("Output: {:?}", String::from_utf8_lossy(&output.stdout)); From 01930556079d60df5066413d5581e472afdb0f38 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:17:02 +0100 Subject: [PATCH 56/77] Fix --- crates/rattler_networking/tests/integration_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 646ab2daf..84f7a52df 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -19,7 +19,7 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p } let output = command .output() - .expect(format!("Failed to execute command {:?}", command).as_str()); + .unwrap_or_else(|_| panic!("Failed to execute command {:?}", command)); if !output.status.success() { eprintln!("Command failed: {:?}", command); eprintln!("Output: {:?}", String::from_utf8_lossy(&output.stdout)); From 8434a39d06e6cfcadfd163889d66a3c618d47471 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:27:10 +0100 Subject: [PATCH 57/77] Test if executables are available in PATH --- .github/workflows/rust-compile.yml | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index c6055ddeb..7f9dd6334 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -118,6 +118,22 @@ jobs: run: | echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT + - name: Set up pixi + if: ${{ !matrix.skip-tests }} + uses: prefix-dev/setup-pixi@v0.8.1 + with: + run-install: false + # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file + post-cleanup: false + + - name: Install minio for integration tests + if: ${{ !matrix.skip-tests }} + run: | + pixi global install minio-client + pixi global install minio-server + which minio + which mc + - name: Build run: > cargo build @@ -138,20 +154,6 @@ jobs: with: tool: cargo-nextest - - name: Set up pixi - if: ${{ !matrix.skip-tests }} - uses: prefix-dev/setup-pixi@v0.8.1 - with: - run-install: false - # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file - post-cleanup: false - - - name: Install minio for integration tests - if: ${{ !matrix.skip-tests }} - run: | - pixi global install minio-client - pixi global install minio-server - - name: Start minio server if: ${{ !matrix.skip-tests }} run: | From c71f413e371d97d26e9f1a95e9b8aa200192fb14 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Sun, 12 Jan 2025 21:32:53 +0100 Subject: [PATCH 58/77] Test with mc directly again --- .github/workflows/rust-compile.yml | 32 +++++++++---------- .../tests/integration_test.rs | 6 ++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 7f9dd6334..1cd4d76ed 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -118,22 +118,6 @@ jobs: run: | echo "CARGO_BUILD_OPTIONS=${CARGO_BUILD_OPTIONS} --no-default-features --features rustls-tls" >> $GITHUB_OUTPUT - - name: Set up pixi - if: ${{ !matrix.skip-tests }} - uses: prefix-dev/setup-pixi@v0.8.1 - with: - run-install: false - # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file - post-cleanup: false - - - name: Install minio for integration tests - if: ${{ !matrix.skip-tests }} - run: | - pixi global install minio-client - pixi global install minio-server - which minio - which mc - - name: Build run: > cargo build @@ -154,10 +138,24 @@ jobs: with: tool: cargo-nextest + - name: Set up pixi + if: ${{ !matrix.skip-tests }} + uses: prefix-dev/setup-pixi@v0.8.1 + with: + run-install: false + # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file + post-cleanup: false + + - name: Install minio for integration tests + if: ${{ !matrix.skip-tests }} + run: | + pixi global install minio-client + pixi global install minio-server + - name: Start minio server if: ${{ !matrix.skip-tests }} run: | - minio${{ startsWith(matrix.os, 'windows') && '.bat' || '' }} server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & - name: Run tests if: ${{ !matrix.skip-tests }} diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index 84f7a52df..bfb93f47d 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -19,9 +19,9 @@ fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::p } let output = command .output() - .unwrap_or_else(|_| panic!("Failed to execute command {:?}", command)); + .unwrap_or_else(|_| panic!("Failed to execute command {command:?}")); if !output.status.success() { - eprintln!("Command failed: {:?}", command); + eprintln!("Command failed: {command:?}"); eprintln!("Output: {:?}", String::from_utf8_lossy(&output.stdout)); eprintln!("Error: {:?}", String::from_utf8_lossy(&output.stderr)); } @@ -47,7 +47,7 @@ fn init_channel() { option_env!("MINIO_PORT").unwrap_or("9000") ); let env = &HashMap::from([("MC_HOST_local", host.as_str())]); - let mc_executable = if cfg!(windows) { "mc.bat" } else { "mc" }; + let mc_executable = "mc"; for bucket in &[ "local/rattler-s3-testing", "local/rattler-s3-testing-public", From 150e3ada1ea3e227668ff92f8775b5007fa3c516 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Mon, 13 Jan 2025 10:57:47 +0100 Subject: [PATCH 59/77] Debug minio windows --- .github/workflows/rust-compile.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 1cd4d76ed..9bf8637f8 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -155,7 +155,10 @@ jobs: - name: Start minio server if: ${{ !matrix.skip-tests }} run: | - minio server --address 0.0.0.0:9000 ${{ runner.temp }}/minio-data & + minio server --help + mkdir -p ${{ runner.temp }}/minio-data + minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & + curl -i localhost:9000 - name: Run tests if: ${{ !matrix.skip-tests }} From c06489e34edfdc158ecf0d135c1216b4dba288f3 Mon Sep 17 00:00:00 2001 From: Daniel Elsner Date: Mon, 13 Jan 2025 11:20:59 +0100 Subject: [PATCH 60/77] Debug minio server --- .github/workflows/rust-compile.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 9bf8637f8..ecb112bf0 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -158,7 +158,8 @@ jobs: minio server --help mkdir -p ${{ runner.temp }}/minio-data minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & - curl -i localhost:9000 + sleep 5 + curl -I http://localhost:9000/minio/health/live - name: Run tests if: ${{ !matrix.skip-tests }} From ba42437c416032627f4345c731e4c6d21587e689 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Mon, 13 Jan 2025 20:24:37 +0100 Subject: [PATCH 61/77] Add cloudflare R2 integration test --- .github/workflows/rust-compile.yml | 2 + crates/rattler/src/cli/auth.rs | 2 +- .../rattler_networking/src/s3_middleware.rs | 8 +-- .../tests/integration_test.rs | 65 ++++++++++++++++++- 4 files changed, 70 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index ecb112bf0..0c3bf827b 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -165,6 +165,8 @@ jobs: if: ${{ !matrix.skip-tests }} env: GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }} + RATTLER_TEST_R2_ACCESS_KEY_ID: ${{ secrets.RATTLER_TEST_R2_ACCESS_KEY_ID }} + RATTLER_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.RATTLER_TEST_R2_SECRET_ACCESS_KEY }} run: > cargo nextest run --workspace diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index acf3d899c..dc5ef1187 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -26,7 +26,7 @@ pub struct LoginArgs { conda_token: Option, /// The S3 access key ID - #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["host", "token", "username", "password", "conda_token"])] + #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["token", "username", "password", "conda_token"])] s3_access_key_id: Option, /// The S3 secret access key diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 66dde00b2..833a0630f 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -5,7 +5,7 @@ use aws_config::BehaviorVersion; use aws_sdk_s3::presigning::PresigningConfig; use reqwest::{Request, Response}; use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; -use tracing::debug; +use tracing::info; use url::Url; use crate::{Authentication, AuthenticationStorage}; @@ -43,7 +43,7 @@ pub struct S3Middleware { impl S3Middleware { /// Create a new S3 middleware. pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { - debug!("Creating S3 middleware using {:?}", config); + info!("Creating S3 middleware using {:?}", config); Self { s3: S3::new(config, auth_storage), } @@ -325,7 +325,7 @@ region = eu-central-1 #[tokio::test] async fn test_presigned_s3_request_custom_config() { let temp_dir = tempdir().unwrap(); - let aws_config = r#" + let credentials = r#" { "s3://rattler-s3-testing/my-channel": { "S3Credentials": { @@ -336,7 +336,7 @@ region = eu-central-1 } "#; let credentials_path = temp_dir.path().join("credentials.json"); - std::fs::write(&credentials_path, aws_config).unwrap(); + std::fs::write(&credentials_path, credentials).unwrap(); let s3 = S3::new( S3Config::Custom { endpoint_url: Url::parse("http://localhost:9000").unwrap(), diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index bfb93f47d..c4a7cf052 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -119,7 +119,7 @@ async fn test_minio_download_repodata( #[allow(unused_variables)] init_channel: (), ) { let temp_dir = tempdir().unwrap(); - let aws_config = r#" + let credentials = r#" { "s3://rattler-s3-testing/my-channel": { "S3Credentials": { @@ -129,7 +129,7 @@ async fn test_minio_download_repodata( } }"#; let credentials_path = temp_dir.path().join("credentials.json"); - std::fs::write(&credentials_path, aws_config).unwrap(); + std::fs::write(&credentials_path, credentials).unwrap(); let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); let middleware = S3Middleware::new( S3Config::Custom { @@ -256,3 +256,64 @@ async fn test_minio_download_aws_profile_public( let body = result.text().await.unwrap(); assert!(body.contains("test-package-0.1-0.tar.bz2")); } + +#[rstest] +#[tokio::test] +async fn test_cloudflare_r2_download_repodata() { + let temp_dir = tempdir().unwrap(); + + let r2_access_key_id = std::env::var("RATTLER_TEST_R2_ACCESS_KEY_ID").ok(); + let r2_secret_access_key = std::env::var("RATTLER_TEST_R2_SECRET_ACCESS_KEY").ok(); + if r2_access_key_id.is_none() || r2_secret_access_key.is_none() { + eprintln!( + "Skipping test as RATTLER_TEST_R2_ACCESS_KEY_ID or RATTLER_TEST_R2_SECRET_ACCESS_KEY is not set" + ); + return; + } + + let r2_access_key_id = r2_access_key_id.unwrap(); + let r2_secret_access_key = r2_secret_access_key.unwrap(); + + let credentials = format!( + r#" + {{ + "s3://rattler-s3-testing/channel": {{ + "S3Credentials": {{ + "access_key_id": "{}", + "secret_access_key": "{}" + }} + }} + }} + "#, + r2_access_key_id, r2_secret_access_key + ); + + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, credentials).unwrap(); + + let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse("https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com").unwrap(), + region: "auto".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("my-webserver-0.1.0-pyh4616a5c_0.conda"), "body does not contain package: {}", body); +} From 90ce8c23d2ed14f139ad1551f599cc7483f0090a Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Mon, 13 Jan 2025 20:30:43 +0100 Subject: [PATCH 62/77] fix --- crates/rattler_networking/tests/integration_test.rs | 11 +++++++++-- py-rattler/pixi.toml | 10 +++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index c4a7cf052..aad5a7d72 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -294,7 +294,10 @@ async fn test_cloudflare_r2_download_repodata() { let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); let middleware = S3Middleware::new( S3Config::Custom { - endpoint_url: Url::parse("https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com").unwrap(), + endpoint_url: Url::parse( + "https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com", + ) + .unwrap(), region: "auto".into(), force_path_style: true, }, @@ -315,5 +318,9 @@ async fn test_cloudflare_r2_download_repodata() { assert_eq!(result.status(), 200); let body = result.text().await.unwrap(); - assert!(body.contains("my-webserver-0.1.0-pyh4616a5c_0.conda"), "body does not contain package: {}", body); + assert!( + body.contains("my-webserver-0.1.0-pyh4616a5c_0.conda"), + "body does not contain package: {}", + body + ); } diff --git a/py-rattler/pixi.toml b/py-rattler/pixi.toml index 290d248e9..f580c49bb 100644 --- a/py-rattler/pixi.toml +++ b/py-rattler/pixi.toml @@ -41,19 +41,19 @@ typer = "*" types-networkx = "*" [feature.test.tasks] -test = { cmd = "pytest --doctest-modules", depends_on = ["build"] } +test = { cmd = "pytest --doctest-modules", depends-on = ["build"] } fmt-python = "ruff format rattler examples tests" fmt-rust = "cargo fmt --all" lint-python = "ruff check ." lint-rust = "cargo clippy --all" -fmt = { depends_on = ["fmt-python", "fmt-rust"] } -lint = { depends_on = ["type-check", "lint-python", "lint-rust"] } -type-check = { cmd = "mypy", depends_on = ["build"] } +fmt = { depends-on = ["fmt-python", "fmt-rust"] } +lint = { depends-on = ["type-check", "lint-python", "lint-rust"] } +type-check = { cmd = "mypy", depends-on = ["build"] } # checks for the CI fmt-rust-check = "cargo fmt --all --check" fmt-python-check = "ruff format rattler examples tests --diff" -fmt-check = { depends_on = ["fmt-python-check", "fmt-rust-check"] } +fmt-check = { depends-on = ["fmt-python-check", "fmt-rust-check"] } [feature.docs.dependencies] mkdocs = ">=1.5.3,<2" From 120c76291f3dbeb9aec947c52e437af704c7ebbc Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 10:03:52 +0100 Subject: [PATCH 63/77] fix --- crates/rattler_networking/tests/integration_test.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index aad5a7d72..d643f46c4 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -264,7 +264,11 @@ async fn test_cloudflare_r2_download_repodata() { let r2_access_key_id = std::env::var("RATTLER_TEST_R2_ACCESS_KEY_ID").ok(); let r2_secret_access_key = std::env::var("RATTLER_TEST_R2_SECRET_ACCESS_KEY").ok(); - if r2_access_key_id.is_none() || r2_secret_access_key.is_none() { + if r2_access_key_id.is_none() + || r2_access_key_id.clone().unwrap().is_empty() + || r2_secret_access_key.is_none() + || r2_secret_access_key.clone().unwrap().is_empty() + { eprintln!( "Skipping test as RATTLER_TEST_R2_ACCESS_KEY_ID or RATTLER_TEST_R2_SECRET_ACCESS_KEY is not set" ); From efa6625c9c59876bb747eace02afa77fcdcc3711 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Tue, 14 Jan 2025 18:17:01 +0100 Subject: [PATCH 64/77] wip auth storage --- crates/rattler_networking/src/s3_middleware.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 833a0630f..cfb9a68d7 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -201,7 +201,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_endpoint_url() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -228,7 +228,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_aws() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -277,7 +277,7 @@ region = eu-central-1 async fn test_presigned_s3_request_custom_config_from_env( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), @@ -301,7 +301,7 @@ region = eu-central-1 #[rstest] #[tokio::test] async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default().unwrap()); let presigned = async_with_vars( [ ("AWS_ENDPOINT_URL", Some("http://localhost:9000")), @@ -369,7 +369,7 @@ region = eu-central-1 region: "eu-central-1".into(), force_path_style: true, }, - AuthenticationStorage::new(), // empty auth storage + AuthenticationStorage::empty(), // empty auth storage ); let presigned = s3 @@ -390,7 +390,7 @@ region = eu-central-1 async fn test_presigned_s3_request_public_bucket_aws( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), From 53484366dff714e133e9fc48a756b6dc8249e513 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 14:45:16 +0100 Subject: [PATCH 65/77] fix error, add sso feature --- Cargo.toml | 6 ++++-- crates/rattler_networking/src/s3_middleware.rs | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 641f176f7..c560026bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,11 +72,13 @@ getrandom = { version = "0.2.15", default-features = false } glob = "0.3.2" google-cloud-auth = { version = "0.17.2", default-features = false } google-cloud-token = "0.1.2" -aws-config = { version = "1.5.13", default-features = false, features = [ +aws-config = { version = "1.5.14", default-features = false, features = [ "rt-tokio", "rustls", + "sso", ] } -aws-sdk-s3 = { version = "1.68.0", default-features = false, features = [ +aws-sdk-sso = { version = "1.54.0", default-features = false } +aws-sdk-s3 = { version = "1.69.0", default-features = false, features = [ "rt-tokio", "rustls", "sigv4a", diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index cfb9a68d7..f4903801f 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -150,12 +150,9 @@ impl S3 { async fn generate_presigned_s3_url(&self, url: Url) -> MiddlewareResult { let client = self.create_s3_client(Some(url.clone())).await?; - let bucket_name = url.host_str().ok_or_else(|| { - reqwest_middleware::Error::middleware(std::io::Error::new( - std::io::ErrorKind::Other, - "host should be present in S3 URL", - )) - })?; + let bucket_name = url + .host_str() + .ok_or_else(|| anyhow::anyhow!("host should be present in S3 URL"))?; let key = url.path().strip_prefix("/").unwrap(); let builder = client.get_object().bucket(bucket_name).key(key); From 321a16d967a772b081f5502cdab5aa8574de50ff Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 15:27:20 +0100 Subject: [PATCH 66/77] fix --- Cargo.toml | 1 - crates/rattler_networking/Cargo.toml | 2 +- .../rattler_networking/src/s3_middleware.rs | 35 +++++++++++-------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c560026bf..e8190d37d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,6 @@ aws-config = { version = "1.5.14", default-features = false, features = [ "rustls", "sso", ] } -aws-sdk-sso = { version = "1.54.0", default-features = false } aws-sdk-s3 = { version = "1.69.0", default-features = false, features = [ "rt-tokio", "rustls", diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index d29893d53..1051b893a 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true readme.workspace = true [features] -default = ["native-tls", "s3"] +default = ["native-tls"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index f4903801f..c66df98a0 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -130,15 +130,20 @@ impl S3 { .await; } } - let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); - // Infer if we expect path-style addressing from the endpoint URL. - if let Some(endpoint_url) = sdk_config.endpoint_url() { - // If the endpoint URL is localhost, we probably have to use path-style addressing. - // There are certainly more edge cases, but this is a valid start to make the - // integration tests with minIO work. - // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 - if endpoint_url.starts_with("http://localhost") { - s3_config_builder = s3_config_builder.force_path_style(true); + #[cfg(not(test))] + let s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + #[cfg(test)] + { + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + // Infer if we expect path-style addressing from the endpoint URL. + if let Some(endpoint_url) = sdk_config.endpoint_url() { + // If the endpoint URL is localhost, we probably have to use path-style addressing. + // There are certainly more edge cases, but this is a valid start to make the + // integration tests with minIO work. + // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 + if endpoint_url.starts_with("http://localhost") { + s3_config_builder = s3_config_builder.force_path_style(true); + } } } let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); @@ -198,7 +203,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_endpoint_url() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -225,7 +230,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_aws() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -274,7 +279,7 @@ region = eu-central-1 async fn test_presigned_s3_request_custom_config_from_env( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), @@ -298,7 +303,7 @@ region = eu-central-1 #[rstest] #[tokio::test] async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default().unwrap()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); let presigned = async_with_vars( [ ("AWS_ENDPOINT_URL", Some("http://localhost:9000")), @@ -366,7 +371,7 @@ region = eu-central-1 region: "eu-central-1".into(), force_path_style: true, }, - AuthenticationStorage::empty(), // empty auth storage + AuthenticationStorage::new(), // empty auth storage ); let presigned = s3 @@ -387,7 +392,7 @@ region = eu-central-1 async fn test_presigned_s3_request_public_bucket_aws( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), From fb7352da46cc37af6b41210d83b6bee6567b5029 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 15:36:57 +0100 Subject: [PATCH 67/77] fix --- crates/rattler_networking/src/s3_middleware.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index c66df98a0..13cc921df 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -130,22 +130,20 @@ impl S3 { .await; } } - #[cfg(not(test))] - let s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); - #[cfg(test)] - { + let s3_config_builder =if cfg!(not(test)) { + aws_sdk_s3::config::Builder::from(&sdk_config) + } else { let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); // Infer if we expect path-style addressing from the endpoint URL. if let Some(endpoint_url) = sdk_config.endpoint_url() { // If the endpoint URL is localhost, we probably have to use path-style addressing. - // There are certainly more edge cases, but this is a valid start to make the - // integration tests with minIO work. // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 if endpoint_url.starts_with("http://localhost") { s3_config_builder = s3_config_builder.force_path_style(true); } } - } + s3_config_builder + }; let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); Ok(client) } From a365c8c389183d5696237d0d8a91bcc9adc872f6 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 15:38:07 +0100 Subject: [PATCH 68/77] fmt --- crates/rattler_networking/src/s3_middleware.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 13cc921df..4793576fc 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -130,7 +130,7 @@ impl S3 { .await; } } - let s3_config_builder =if cfg!(not(test)) { + let s3_config_builder = if cfg!(not(test)) { aws_sdk_s3::config::Builder::from(&sdk_config) } else { let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); From ef60b64a18cba6755e066d9548978c933868b6ec Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 16:01:51 +0100 Subject: [PATCH 69/77] whatever, don't annoy me --- .../rattler_networking/src/s3_middleware.rs | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index 4793576fc..c53c40cc3 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -130,20 +130,15 @@ impl S3 { .await; } } - let s3_config_builder = if cfg!(not(test)) { - aws_sdk_s3::config::Builder::from(&sdk_config) - } else { - let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); - // Infer if we expect path-style addressing from the endpoint URL. - if let Some(endpoint_url) = sdk_config.endpoint_url() { - // If the endpoint URL is localhost, we probably have to use path-style addressing. - // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 - if endpoint_url.starts_with("http://localhost") { - s3_config_builder = s3_config_builder.force_path_style(true); - } + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + // Infer if we expect path-style addressing from the endpoint URL. + if let Some(endpoint_url) = sdk_config.endpoint_url() { + // If the endpoint URL is localhost, we probably have to use path-style addressing. + // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 + if endpoint_url.starts_with("http://localhost") { + s3_config_builder = s3_config_builder.force_path_style(true); } - s3_config_builder - }; + } let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); Ok(client) } From 369c181d6456fa0f13e21b061f7fdea0f59146ca Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 16 Jan 2025 16:20:55 +0100 Subject: [PATCH 70/77] empty commit From 73bdffb9bdd0984f5cb8a0efebd3e97159dd2b1e Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 16:16:26 +0100 Subject: [PATCH 71/77] fix after merge --- .../rattler_networking/src/s3_middleware.rs | 20 +++++++++----- .../tests/integration_test.rs | 26 +++++++++---------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs index c53c40cc3..b652b36fa 100644 --- a/crates/rattler_networking/src/s3_middleware.rs +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -189,6 +189,10 @@ impl Middleware for S3Middleware { #[cfg(test)] mod tests { + use std::sync::Arc; + + use crate::authentication_storage::backends::file::FileStorage; + use super::*; use rstest::{fixture, rstest}; use temp_env::async_with_vars; @@ -196,7 +200,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_endpoint_url() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -223,7 +227,7 @@ mod tests { #[tokio::test] async fn test_presigned_s3_request_aws() { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_ACCESS_KEY_ID", Some("minioadmin")), @@ -272,7 +276,7 @@ region = eu-central-1 async fn test_presigned_s3_request_custom_config_from_env( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), @@ -296,7 +300,7 @@ region = eu-central-1 #[rstest] #[tokio::test] async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::default()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_ENDPOINT_URL", Some("http://localhost:9000")), @@ -332,13 +336,15 @@ region = eu-central-1 "#; let credentials_path = temp_dir.path().join("credentials.json"); std::fs::write(&credentials_path, credentials).unwrap(); + let mut store = AuthenticationStorage::empty(); + store.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); let s3 = S3::new( S3Config::Custom { endpoint_url: Url::parse("http://localhost:9000").unwrap(), region: "eu-central-1".into(), force_path_style: true, }, - AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(), + store, ); let presigned = s3 @@ -364,7 +370,7 @@ region = eu-central-1 region: "eu-central-1".into(), force_path_style: true, }, - AuthenticationStorage::new(), // empty auth storage + AuthenticationStorage::empty(), // empty auth storage ); let presigned = s3 @@ -385,7 +391,7 @@ region = eu-central-1 async fn test_presigned_s3_request_public_bucket_aws( aws_config: (TempDir, std::path::PathBuf), ) { - let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::new()); + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); let presigned = async_with_vars( [ ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index d643f46c4..f7e39382a 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use rattler_networking::{ - s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware, + authentication_storage::backends::file::FileStorage, s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware }; use reqwest::Client; use rstest::*; @@ -130,7 +130,8 @@ async fn test_minio_download_repodata( }"#; let credentials_path = temp_dir.path().join("credentials.json"); std::fs::write(&credentials_path, credentials).unwrap(); - let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); + let mut auth_storage = AuthenticationStorage::empty(); + auth_storage.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); let middleware = S3Middleware::new( S3Config::Custom { endpoint_url: Url::parse(minio_host).unwrap(), @@ -142,7 +143,7 @@ async fn test_minio_download_repodata( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) .with(middleware) .build(); @@ -163,7 +164,7 @@ async fn test_minio_download_repodata_public( minio_host: &str, #[allow(unused_variables)] init_channel: (), ) { - let auth_storage = AuthenticationStorage::new(); // empty storage + let auth_storage = AuthenticationStorage::empty(); let middleware = S3Middleware::new( S3Config::Custom { endpoint_url: Url::parse(minio_host).unwrap(), @@ -175,7 +176,7 @@ async fn test_minio_download_repodata_public( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) .with(middleware) .build(); @@ -196,12 +197,11 @@ async fn test_minio_download_repodata_aws_profile( aws_config: (TempDir, std::path::PathBuf), #[allow(unused_variables)] init_channel: (), ) { - let auth_storage = AuthenticationStorage::new(); // empty storage - let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); + let middleware = S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::empty()); let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(AuthenticationStorage::empty()))) .with(middleware) .build(); @@ -230,12 +230,11 @@ async fn test_minio_download_aws_profile_public( aws_config: (TempDir, std::path::PathBuf), #[allow(unused_variables)] init_channel: (), ) { - let auth_storage = AuthenticationStorage::new(); // empty storage - let middleware = S3Middleware::new(S3Config::FromAWS, auth_storage.clone()); + let middleware = S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::empty()); let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults().unwrap())) .with(middleware) .build(); let result = async_with_vars( @@ -295,7 +294,8 @@ async fn test_cloudflare_r2_download_repodata() { let credentials_path = temp_dir.path().join("credentials.json"); std::fs::write(&credentials_path, credentials).unwrap(); - let auth_storage = AuthenticationStorage::from_file(credentials_path.as_path()).unwrap(); + let mut auth_storage = AuthenticationStorage::empty(); + auth_storage.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); let middleware = S3Middleware::new( S3Config::Custom { endpoint_url: Url::parse( @@ -310,7 +310,7 @@ async fn test_cloudflare_r2_download_repodata() { let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::new(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) .with(middleware) .build(); From 87f88588da9e2332d7b4f9caff12dc5c6bc6e72f Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 16:19:11 +0100 Subject: [PATCH 72/77] fmt --- .../tests/integration_test.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs index f7e39382a..78a81d54a 100644 --- a/crates/rattler_networking/tests/integration_test.rs +++ b/crates/rattler_networking/tests/integration_test.rs @@ -1,7 +1,8 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use rattler_networking::{ - authentication_storage::backends::file::FileStorage, s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, S3Middleware + authentication_storage::backends::file::FileStorage, s3_middleware::S3Config, + AuthenticationMiddleware, AuthenticationStorage, S3Middleware, }; use reqwest::Client; use rstest::*; @@ -143,7 +144,9 @@ async fn test_minio_download_repodata( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) .with(middleware) .build(); @@ -176,7 +179,9 @@ async fn test_minio_download_repodata_public( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) .with(middleware) .build(); @@ -201,7 +206,9 @@ async fn test_minio_download_repodata_aws_profile( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(AuthenticationStorage::empty()))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + AuthenticationStorage::empty(), + ))) .with(middleware) .build(); @@ -234,7 +241,9 @@ async fn test_minio_download_aws_profile_public( let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults().unwrap())) + .with_arc(Arc::new( + AuthenticationMiddleware::from_env_and_defaults().unwrap(), + )) .with(middleware) .build(); let result = async_with_vars( @@ -310,7 +319,9 @@ async fn test_cloudflare_r2_download_repodata() { let download_client = Client::builder().no_gzip().build().unwrap(); let download_client = reqwest_middleware::ClientBuilder::new(download_client) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage(auth_storage))) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) .with(middleware) .build(); From e1fbc8265e48336bc96e8997e04c621ac617a661 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 16:25:30 +0100 Subject: [PATCH 73/77] does pwsh work? --- .github/workflows/rust-compile.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 0c3bf827b..286e11416 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -153,13 +153,24 @@ jobs: pixi global install minio-server - name: Start minio server - if: ${{ !matrix.skip-tests }} + if: ${{ !matrix.skip-tests && !startsWith(matrix.os, 'windows-') }} run: | minio server --help mkdir -p ${{ runner.temp }}/minio-data minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & sleep 5 - curl -I http://localhost:9000/minio/health/live + + - name: Start minio server + if: ${{ !matrix.skip-tests && startsWith(matrix.os, 'windows-') }} + shell: powershell + run: | + minio server --help + mkdir ${{ runner.temp }}\minio-data + Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -Wait -NoNewWindow + Start-Sleep -Seconds 5 + + - name: Test minio server connection + run: curl -I http://localhost:9000/minio/health/live - name: Run tests if: ${{ !matrix.skip-tests }} From ffae735736e51c5047120d3a3fc6537baed6cf34 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 16:40:48 +0100 Subject: [PATCH 74/77] ? --- .github/workflows/rust-compile.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 286e11416..b83f937f9 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -166,10 +166,11 @@ jobs: run: | minio server --help mkdir ${{ runner.temp }}\minio-data - Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -Wait -NoNewWindow + Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -NoNewWindow Start-Sleep -Seconds 5 - name: Test minio server connection + if: ${{ !matrix.skip-tests }} run: curl -I http://localhost:9000/minio/health/live - name: Run tests From ca7e4d7dfe11f169580464b79b80ef7859cf7d6d Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 16:43:09 +0100 Subject: [PATCH 75/77] ? --- .github/workflows/rust-compile.yml | 57 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index b83f937f9..7ce5168af 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -152,26 +152,26 @@ jobs: pixi global install minio-client pixi global install minio-server - - name: Start minio server - if: ${{ !matrix.skip-tests && !startsWith(matrix.os, 'windows-') }} - run: | - minio server --help - mkdir -p ${{ runner.temp }}/minio-data - minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & - sleep 5 - - - name: Start minio server - if: ${{ !matrix.skip-tests && startsWith(matrix.os, 'windows-') }} - shell: powershell - run: | - minio server --help - mkdir ${{ runner.temp }}\minio-data - Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -NoNewWindow - Start-Sleep -Seconds 5 - - - name: Test minio server connection - if: ${{ !matrix.skip-tests }} - run: curl -I http://localhost:9000/minio/health/live + # - name: Start minio server + # if: ${{ !matrix.skip-tests && !startsWith(matrix.os, 'windows-') }} + # run: | + # minio server --help + # mkdir -p ${{ runner.temp }}/minio-data + # minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & + # sleep 5 + + # - name: Start minio server + # if: ${{ !matrix.skip-tests && startsWith(matrix.os, 'windows-') }} + # shell: powershell + # run: | + # minio server --help + # mkdir ${{ runner.temp }}\minio-data + # Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -NoNewWindow + # Start-Sleep -Seconds 5 + + # - name: Test minio server connection + # if: ${{ !matrix.skip-tests }} + # run: curl -I http://localhost:9000/minio/health/live - name: Run tests if: ${{ !matrix.skip-tests }} @@ -179,12 +179,17 @@ jobs: GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }} RATTLER_TEST_R2_ACCESS_KEY_ID: ${{ secrets.RATTLER_TEST_R2_ACCESS_KEY_ID }} RATTLER_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.RATTLER_TEST_R2_SECRET_ACCESS_KEY }} - run: > - cargo nextest run - --workspace - --features ${{ env.DEFAULT_FEATURES }} - --target ${{ matrix.target }} - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} + run: | + minio server --help + mkdir -p ${{ runner.temp }}/minio-data + minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & + sleep 5 + curl -I http://localhost:9000/minio/health/live + cargo nextest run \ + --workspace \ + --features ${{ env.DEFAULT_FEATURES }} \ + --target ${{ matrix.target }} \ + ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} \ ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} - name: Run doctests From 73eed8a7d50de3e5e766d555052bfe2ba1225049 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 17:01:42 +0100 Subject: [PATCH 76/77] ? --- .github/workflows/rust-compile.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 7ce5168af..3cd401a35 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -185,12 +185,7 @@ jobs: minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & sleep 5 curl -I http://localhost:9000/minio/health/live - cargo nextest run \ - --workspace \ - --features ${{ env.DEFAULT_FEATURES }} \ - --target ${{ matrix.target }} \ - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} \ - ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} + cargo nextest run --workspace --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} - name: Run doctests if: ${{ !matrix.skip-tests }} From 11f07b021c2a49b81c97c5999939648ad6c27430 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Fri, 17 Jan 2025 21:38:20 +0100 Subject: [PATCH 77/77] fix --- .github/workflows/rust-compile.yml | 21 --------------------- py-rattler/src/networking/client.rs | 18 +++--------------- 2 files changed, 3 insertions(+), 36 deletions(-) diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index 3cd401a35..47c49c0fe 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -152,27 +152,6 @@ jobs: pixi global install minio-client pixi global install minio-server - # - name: Start minio server - # if: ${{ !matrix.skip-tests && !startsWith(matrix.os, 'windows-') }} - # run: | - # minio server --help - # mkdir -p ${{ runner.temp }}/minio-data - # minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & - # sleep 5 - - # - name: Start minio server - # if: ${{ !matrix.skip-tests && startsWith(matrix.os, 'windows-') }} - # shell: powershell - # run: | - # minio server --help - # mkdir ${{ runner.temp }}\minio-data - # Start-Process "minio" -ArgumentList "server --address 127.0.0.1:9000 ${{ runner.temp }}\minio-data" -NoNewWindow - # Start-Sleep -Seconds 5 - - # - name: Test minio server connection - # if: ${{ !matrix.skip-tests }} - # run: curl -I http://localhost:9000/minio/health/live - - name: Run tests if: ${{ !matrix.skip-tests }} env: diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index bd18b2357..6c735b46a 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -1,13 +1,11 @@ use crate::{error::PyRattlerError, networking::middleware::PyMiddleware}; use pyo3::{pyclass, pymethods, PyResult}; use rattler_networking::{ - s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, - MirrorMiddleware, OciMiddleware, S3Middleware, + AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, + OciMiddleware, S3Middleware, }; use reqwest_middleware::ClientWithMiddleware; -use super::middleware::PyS3Config; - #[pyclass] #[repr(transparent)] #[derive(Clone)] @@ -40,18 +38,8 @@ impl PyClientWithMiddleware { client = client.with(GCSMiddleware::from(middleware)); } PyMiddleware::S3(middleware) => { - let PyS3Config { custom } = middleware.s3_config; - let s3_config = if let Some(config) = custom { - S3Config::Custom { - endpoint_url: config.endpoint_url, - region: config.region, - force_path_style: config.force_path_style, - } - } else { - S3Config::FromAWS - }; client = client.with(S3Middleware::new( - s3_config, + middleware.s3_config.into(), AuthenticationStorage::from_env_and_defaults() .map_err(PyRattlerError::from)?, ));