diff --git a/.github/workflows/rust-compile.yml b/.github/workflows/rust-compile.yml index e60144ae2..47c49c0fe 100644 --- a/.github/workflows/rust-compile.yml +++ b/.github/workflows/rust-compile.yml @@ -1,6 +1,6 @@ on: push: - branches: [ main ] + branches: [main] pull_request: paths: # When we change pyproject.toml, we want to ensure that the maturin builds still work @@ -22,7 +22,7 @@ env: RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" CARGO_TERM_COLOR: always - DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c + DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c,s3 jobs: check-rustdoc-links: @@ -54,7 +54,7 @@ jobs: build: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} - needs: [ format_and_lint ] + needs: [format_and_lint] strategy: fail-fast: false matrix: @@ -62,20 +62,19 @@ jobs: - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-22.04 } - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest, skip-tests: true } - { name: "Linux-arm", target: arm-unknown-linux-musleabi, os: ubuntu-latest, use-cross: true, skip-tests: true } - # - { name: "Linux-mips", target: mips-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mipsel", target: mipsel-unknown-linux-musl, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64", target: mips64-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } # - { name: "Linux-mips64el", target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true } -# - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } + # - { name: "Linux-powerpc", target: powerpc-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - { name: "Linux-powerpc64", target: powerpc64-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - { name: "Linux-s390x", target: s390x-unknown-linux-gnu, os: ubuntu-latest, use-cross: true, skip-tests: true } - { name: "macOS-x86_64", target: x86_64-apple-darwin, os: macOS-latest } - - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest, skip-tests: true } + - { name: "macOS-aarch64", target: aarch64-apple-darwin, os: macOS-latest } - { name: "Windows-x86_64", target: x86_64-pc-windows-msvc, os: windows-latest } - { name: "Windows-aarch64", target: aarch64-pc-windows-msvc, os: windows-latest, skip-tests: true } @@ -121,11 +120,11 @@ jobs: - name: Build run: > - cargo build - --all-targets + cargo build + --all-targets --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} + ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} - name: Disable testing the tools crate if cross compiling id: test-options @@ -139,17 +138,33 @@ jobs: with: tool: cargo-nextest + - name: Set up pixi + if: ${{ !matrix.skip-tests }} + uses: prefix-dev/setup-pixi@v0.8.1 + with: + run-install: false + # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file + post-cleanup: false + + - name: Install minio for integration tests + if: ${{ !matrix.skip-tests }} + run: | + pixi global install minio-client + pixi global install minio-server + - name: Run tests if: ${{ !matrix.skip-tests }} env: GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }} - run: > - cargo nextest run - --workspace - --features ${{ env.DEFAULT_FEATURES }} - --target ${{ matrix.target }} - ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} - ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} + RATTLER_TEST_R2_ACCESS_KEY_ID: ${{ secrets.RATTLER_TEST_R2_ACCESS_KEY_ID }} + RATTLER_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.RATTLER_TEST_R2_SECRET_ACCESS_KEY }} + run: | + minio server --help + mkdir -p ${{ runner.temp }}/minio-data + minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data & + sleep 5 + curl -I http://localhost:9000/minio/health/live + cargo nextest run --workspace --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }} - name: Run doctests if: ${{ !matrix.skip-tests }} diff --git a/Cargo.toml b/Cargo.toml index ec3db1cfc..0b85df429 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,16 @@ getrandom = { version = "0.2.15", default-features = false } glob = "0.3.2" google-cloud-auth = { version = "0.17.2", default-features = false } google-cloud-token = "0.1.2" +aws-config = { version = "1.5.14", default-features = false, features = [ + "rt-tokio", + "rustls", + "sso", +] } +aws-sdk-s3 = { version = "1.69.0", default-features = false, features = [ + "rt-tokio", + "rustls", + "sigv4a", +] } hex = "0.4.3" hex-literal = "0.4.1" http = "1.2" @@ -142,7 +152,7 @@ sysinfo = "0.33.1" tar = "0.4.43" tempdir = "0.3.7" tempfile = "3.15.0" -temp-env = "0.3.6" +temp-env = { version = "0.3.6", features = ["async_closure"] } test-log = "0.2.16" thiserror = "2.0" tokio = { version = "1.42.0", default-features = false } diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index d59a34fbc..82603a36d 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -29,7 +29,7 @@ indicatif = { workspace = true } once_cell = { workspace = true } rattler = { path="../rattler", version = "0.28.12", default-features = false, features = ["indicatif"] } rattler_conda_types = { path="../rattler_conda_types", version = "0.29.10", default-features = false } -rattler_networking = { path="../rattler_networking", version = "0.21.10", default-features = false, features = ["gcs"] } +rattler_networking = { path="../rattler_networking", version = "0.21.10", default-features = false, features = ["gcs", "s3"] } rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.32", default-features = false, features = ["gateway"] } rattler_solve = { path="../rattler_solve", version = "1.3.4", default-features = false, features = ["resolvo", "libsolv_c"] } rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.2.0", default-features = false } diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 7ab8e7fd0..c814e9aa8 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -21,7 +21,9 @@ use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform, PrefixRecord, RepoDataRecord, Version, }; -use rattler_networking::AuthenticationMiddleware; +use rattler_networking::{ + s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage, +}; use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig}; use rattler_solve::{ libsolv_c::{self}, @@ -150,6 +152,10 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> { let download_client = reqwest_middleware::ClientBuilder::new(download_client) .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults()?)) .with(rattler_networking::OciMiddleware) + .with(rattler_networking::S3Middleware::new( + S3Config::FromAWS, + AuthenticationStorage::from_env_and_defaults()?, + )) .with(rattler_networking::GCSMiddleware) .build(); diff --git a/crates/rattler/src/cli/auth.rs b/crates/rattler/src/cli/auth.rs index da7830211..5b3e2fb1d 100644 --- a/crates/rattler/src/cli/auth.rs +++ b/crates/rattler/src/cli/auth.rs @@ -26,6 +26,18 @@ pub struct LoginArgs { /// The token to use on anaconda.org / quetz authentication #[clap(long)] conda_token: Option, + + /// The S3 access key ID + #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["token", "username", "password", "conda_token"])] + s3_access_key_id: Option, + + /// The S3 secret access key + #[clap(long, requires_all = ["s3_access_key_id"])] + s3_secret_access_key: Option, + + /// The S3 session token + #[clap(long, requires_all = ["s3_access_key_id"])] + s3_session_token: Option, } #[derive(Parser, Debug)] @@ -73,6 +85,10 @@ pub enum AuthenticationCLIError { #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")] AnacondaOrgBadMethod, + /// Bad authentication method when using S3 + #[error("Authentication with S3 requires a S3 access key ID and a secret access key. Use `--s3-access-key-id` and `--s3-secret-access-key` to provide them")] + S3BadMethod, + /// Wrapper for errors that are generated from the underlying storage system /// (keyring or file system) #[error("Failed to initialize the authentication storage system")] @@ -86,7 +102,7 @@ pub enum AuthenticationCLIError { fn get_url(url: &str) -> Result { // parse as url and extract host without scheme or port - let host = if url.contains("://") { + let host = if url.contains("http://") || url.contains("https://") { url::Url::parse(url)?.host_str().unwrap().to_string() } else { url.to_string() @@ -117,6 +133,15 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti } } else if let Some(token) = args.token { Authentication::BearerToken(token) + } else if let (Some(access_key_id), Some(secret_access_key)) = + (args.s3_access_key_id, args.s3_secret_access_key) + { + let session_token = args.s3_session_token; + Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + } } else { return Err(AuthenticationCLIError::NoAuthenticationMethod); }; @@ -129,6 +154,14 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti return Err(AuthenticationCLIError::AnacondaOrgBadMethod); } + if host.starts_with("s3://") && !matches!(auth, Authentication::S3Credentials { .. }) { + return Err(AuthenticationCLIError::S3BadMethod); + } + + if matches!(auth, Authentication::S3Credentials { .. }) && !host.starts_with("s3://") { + return Err(AuthenticationCLIError::S3BadMethod); + } + storage .store(&host, &auth) .map_err(AuthenticationCLIError::StorageError)?; diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 1e36bd80e..52c1b0d66 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -15,6 +15,7 @@ default = ["native-tls"] native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"] rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"] gcs = ["google-cloud-auth", "google-cloud-token"] +s3 = ["aws-config", "aws-sdk-s3"] [dependencies] anyhow = { workspace = true } @@ -25,9 +26,17 @@ chrono = { workspace = true } dirs = { workspace = true } google-cloud-auth = { workspace = true, optional = true } google-cloud-token = { workspace = true, optional = true } +aws-config = { workspace = true, optional = true } +aws-sdk-s3 = { workspace = true, optional = true } http = { workspace = true } itertools = { workspace = true } -keyring = { workspace = true, features = ["apple-native", "windows-native", "async-secret-service", "async-io", "crypto-rust"] } +keyring = { workspace = true, features = [ + "apple-native", + "windows-native", + "async-secret-service", + "async-io", + "crypto-rust", +] } netrc-rs = { workspace = true } reqwest = { workspace = true, features = ["json"] } reqwest-middleware = { workspace = true } @@ -50,3 +59,5 @@ axum = { workspace = true } reqwest-retry = { workspace = true } sha2 = { workspace = true } temp-env = { workspace = true } +rstest = { workspace = true } +rand = { workspace = true } diff --git a/crates/rattler_networking/src/authentication_middleware.rs b/crates/rattler_networking/src/authentication_middleware.rs index 6fdf4ad7d..046e15983 100644 --- a/crates/rattler_networking/src/authentication_middleware.rs +++ b/crates/rattler_networking/src/authentication_middleware.rs @@ -113,7 +113,7 @@ impl AuthenticationMiddleware { .insert(reqwest::header::AUTHORIZATION, header_value); Ok(req) } - Authentication::CondaToken(_) => Ok(req), + Authentication::CondaToken(_) | Authentication::S3Credentials { .. } => Ok(req), } } else { Ok(req) diff --git a/crates/rattler_networking/src/authentication_storage/authentication.rs b/crates/rattler_networking/src/authentication_storage/authentication.rs index 6939b6675..74f9f3128 100644 --- a/crates/rattler_networking/src/authentication_storage/authentication.rs +++ b/crates/rattler_networking/src/authentication_storage/authentication.rs @@ -19,6 +19,15 @@ pub enum Authentication { }, /// A conda token is sent in the URL as `/t/{TOKEN}/...` CondaToken(String), + /// S3 credentials + S3Credentials { + /// The access key ID to use for S3 authentication + access_key_id: String, + /// The secret access key to use for S3 authentication + secret_access_key: String, + /// The session token to use for S3 authentication + session_token: Option, + }, } /// An error that can occur when parsing an authentication string diff --git a/crates/rattler_networking/src/authentication_storage/storage.rs b/crates/rattler_networking/src/authentication_storage/storage.rs index 92054b50c..a2cd0c85f 100644 --- a/crates/rattler_networking/src/authentication_storage/storage.rs +++ b/crates/rattler_networking/src/authentication_storage/storage.rs @@ -142,6 +142,32 @@ impl AuthenticationStorage { Ok(Some(credentials)) => return Ok((url, Some(credentials))), }; + // S3 protocol URLs need to be treated separately since they follow a different schema + if url.scheme() == "s3" { + let mut current_url = url.clone(); + loop { + match self.get(current_url.as_str()) { + Ok(None) => { + let possible_rest = + current_url.as_str().rsplit_once('/').map(|(rest, _)| rest); + + match possible_rest { + Some(rest) => { + if let Ok(new_url) = Url::parse(rest) { + current_url = new_url; + } else { + return Ok((url, None)); + } + } + _ => return Ok((url, None)), // No more subpaths to check + } + } + Ok(Some(credentials)) => return Ok((url, Some(credentials))), + Err(_) => return Ok((url, None)), + } + } + } + // Check for credentials under e.g. `*.prefix.dev` let Some(mut domain) = url.domain() else { return Ok((url, None)); diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index c8ba7ff18..23c4ffce1 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -11,6 +11,11 @@ pub mod gcs_middleware; #[cfg(feature = "gcs")] pub use gcs_middleware::GCSMiddleware; +#[cfg(feature = "s3")] +pub mod s3_middleware; +#[cfg(feature = "s3")] +pub use s3_middleware::S3Middleware; + pub mod authentication_middleware; pub mod authentication_storage; diff --git a/crates/rattler_networking/src/s3_middleware.rs b/crates/rattler_networking/src/s3_middleware.rs new file mode 100644 index 000000000..b652b36fa --- /dev/null +++ b/crates/rattler_networking/src/s3_middleware.rs @@ -0,0 +1,415 @@ +//! Middleware to handle `s3://` URLs to pull artifacts from an S3 bucket +use anyhow::Error; +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_s3::presigning::PresigningConfig; +use reqwest::{Request, Response}; +use reqwest_middleware::{Middleware, Next, Result as MiddlewareResult}; +use tracing::info; +use url::Url; + +use crate::{Authentication, AuthenticationStorage}; + +/// Configuration for the S3 middleware. +#[derive(Clone, Debug)] +pub enum S3Config { + /// Use the default AWS configuration. + FromAWS, + /// Use a custom configuration. + Custom { + /// The endpoint URL to use for the S3 client. + endpoint_url: Url, + /// The region to use for the S3 client. + region: String, + /// Whether to force path style for the S3 client. + force_path_style: bool, + }, +} + +/// Wrapper around S3 client. +#[derive(Clone, Debug)] +pub struct S3 { + auth_storage: AuthenticationStorage, + config: S3Config, + expiration: std::time::Duration, +} + +/// S3 middleware to authenticate requests. +#[derive(Clone, Debug)] +pub struct S3Middleware { + s3: S3, +} + +impl S3Middleware { + /// Create a new S3 middleware. + pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { + info!("Creating S3 middleware using {:?}", config); + Self { + s3: S3::new(config, auth_storage), + } + } +} + +impl S3 { + /// Create a new S3 client wrapper. + pub fn new(config: S3Config, auth_storage: AuthenticationStorage) -> Self { + Self { + config, + auth_storage, + expiration: std::time::Duration::from_secs(300), + } + } + + /// Create an S3 client. + /// + /// # Arguments + /// + /// * `url` - The S3 URL to obtain authentication information from the authentication storage. + /// Only respected for custom (non-AWS-based) configuration. + pub async fn create_s3_client(&self, url: Option) -> Result { + if let ( + S3Config::Custom { + endpoint_url, + region, + force_path_style, + }, + Some(url), + ) = (self.config.clone(), url) + { + let auth = self.auth_storage.get_by_url(url).unwrap(); // todo + let config_builder = match auth { + ( + _, + Some(Authentication::S3Credentials { + access_key_id, + secret_access_key, + session_token, + }), + ) => { + let sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) + .credentials_provider(aws_sdk_s3::config::Credentials::new( + access_key_id, + secret_access_key, + session_token, + None, + "pixi", + )) + } + (_, Some(_)) => { + return Err(anyhow::anyhow!("unsupported authentication method")); + } + (_, None) => { + tracing::info!("No authentication found, assuming bucket is public"); + let sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .no_credentials() // Turn off request signing + .load() + .await; + aws_sdk_s3::config::Builder::from(&sdk_config) + .endpoint_url(endpoint_url) + .region(aws_sdk_s3::config::Region::new(region)) + .force_path_style(force_path_style) + } + }; + let s3_config = config_builder.build(); + Ok(aws_sdk_s3::Client::from_conf(s3_config)) + } else { + let mut sdk_config = aws_config::defaults(BehaviorVersion::latest()).load().await; + if let Some(credentials_provider) = sdk_config.credentials_provider() { + let creds = credentials_provider.as_ref().provide_credentials().await; + if creds.is_ok() { + tracing::info!("Using AWS credentials from environment via default provider"); + } else { + tracing::warn!("No AWS credentials found, assuming bucket is public"); + sdk_config = aws_config::defaults(BehaviorVersion::latest()) + .no_credentials() + .load() + .await; + } + } + let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&sdk_config); + // Infer if we expect path-style addressing from the endpoint URL. + if let Some(endpoint_url) = sdk_config.endpoint_url() { + // If the endpoint URL is localhost, we probably have to use path-style addressing. + // xref: https://github.com/awslabs/aws-sdk-rust/issues/1230 + if endpoint_url.starts_with("http://localhost") { + s3_config_builder = s3_config_builder.force_path_style(true); + } + } + let client = aws_sdk_s3::Client::from_conf(s3_config_builder.build()); + Ok(client) + } + } + + /// Generate a presigned S3 `GetObject` request. + async fn generate_presigned_s3_url(&self, url: Url) -> MiddlewareResult { + let client = self.create_s3_client(Some(url.clone())).await?; + + let bucket_name = url + .host_str() + .ok_or_else(|| anyhow::anyhow!("host should be present in S3 URL"))?; + let key = url.path().strip_prefix("/").unwrap(); + + let builder = client.get_object().bucket(bucket_name).key(key); + + Url::parse( + builder + .presigned( + PresigningConfig::expires_in(self.expiration) + .map_err(reqwest_middleware::Error::middleware)?, + ) + .await + .map_err(reqwest_middleware::Error::middleware)? + .uri(), + ) + .map_err(reqwest_middleware::Error::middleware) + } +} + +#[async_trait] +impl Middleware for S3Middleware { + /// Create a new authentication middleware for S3. + async fn handle( + &self, + mut req: Request, + extensions: &mut http::Extensions, + next: Next<'_>, + ) -> MiddlewareResult { + if req.url().scheme() == "s3" { + let url = req.url().clone(); + let presigned_url = self.s3.generate_presigned_s3_url(url).await?; + *req.url_mut() = presigned_url.clone(); + } + next.run(req, extensions).await + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::authentication_storage::backends::file::FileStorage; + + use super::*; + use rstest::{fixture, rstest}; + use temp_env::async_with_vars; + use tempfile::{tempdir, TempDir}; + + #[tokio::test] + async fn test_presigned_s3_request_endpoint_url() { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let presigned = async_with_vars( + [ + ("AWS_ACCESS_KEY_ID", Some("minioadmin")), + ("AWS_SECRET_ACCESS_KEY", Some("minioadmin")), + ("AWS_REGION", Some("eu-central-1")), + ("AWS_ENDPOINT_URL", Some("http://custom-aws")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() + }, + ) + .await; + assert!( + presigned.to_string().starts_with( + "http://rattler-s3-testing.custom-aws/my-channel/noarch/repodata.json?" + ), + "Unexpected presigned URL: {presigned:?}" + ); + } + + #[tokio::test] + async fn test_presigned_s3_request_aws() { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let presigned = async_with_vars( + [ + ("AWS_ACCESS_KEY_ID", Some("minioadmin")), + ("AWS_SECRET_ACCESS_KEY", Some("minioadmin")), + ("AWS_REGION", Some("eu-central-1")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() + }, + ) + .await; + assert!(presigned.to_string().starts_with("https://rattler-s3-testing.s3.eu-central-1.amazonaws.com/my-channel/noarch/repodata.json?"), "Unexpected presigned URL: {presigned:?}" + ); + } + + #[fixture] + fn aws_config() -> (TempDir, std::path::PathBuf) { + let temp_dir = tempdir().unwrap(); + let aws_config = r#" +[profile default] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +region = eu-central-1 + +[profile packages] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +endpoint_url = http://localhost:9000 +region = eu-central-1 + +[profile public] +endpoint_url = http://localhost:9000 +region = eu-central-1 +"#; + let aws_config_path = temp_dir.path().join("aws.config"); + std::fs::write(&aws_config_path, aws_config).unwrap(); + (temp_dir, aws_config_path) + } + + #[rstest] + #[tokio::test] + async fn test_presigned_s3_request_custom_config_from_env( + aws_config: (TempDir, std::path::PathBuf), + ) { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let presigned = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("packages")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() + }, + ) + .await; + assert!( + presigned.to_string().contains("localhost:9000"), + "Unexpected presigned URL: {presigned:?}" + ); + } + + #[rstest] + #[tokio::test] + async fn test_presigned_s3_request_env_precedence(aws_config: (TempDir, std::path::PathBuf)) { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let presigned = async_with_vars( + [ + ("AWS_ENDPOINT_URL", Some("http://localhost:9000")), + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() + }, + ) + .await; + assert!( + presigned.to_string().contains("localhost:9000"), + "Unexpected presigned URL: {presigned:?}" + ); + } + + #[tokio::test] + async fn test_presigned_s3_request_custom_config() { + let temp_dir = tempdir().unwrap(); + let credentials = r#" + { + "s3://rattler-s3-testing/my-channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } + } + "#; + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, credentials).unwrap(); + let mut store = AuthenticationStorage::empty(); + store.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); + let s3 = S3::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + store, + ); + + let presigned = s3 + .generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap(); + assert_eq!( + presigned.path(), + "/rattler-s3-testing/my-channel/noarch/repodata.json" + ); + assert_eq!(presigned.scheme(), "http"); + assert_eq!(presigned.host_str().unwrap(), "localhost"); + assert!(presigned.query().unwrap().contains("X-Amz-Credential")); + } + + #[tokio::test] + async fn test_presigned_s3_request_public_bucket() { + let s3 = S3::new( + S3Config::Custom { + endpoint_url: Url::parse("http://localhost:9000").unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + AuthenticationStorage::empty(), // empty auth storage + ); + + let presigned = s3 + .generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap(); + assert!( + presigned.to_string() + == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", + "Unexpected presigned URL: {presigned:?}" + ); + } + + #[rstest] + #[tokio::test] + async fn test_presigned_s3_request_public_bucket_aws( + aws_config: (TempDir, std::path::PathBuf), + ) { + let s3 = S3::new(S3Config::FromAWS, AuthenticationStorage::empty()); + let presigned = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("public")), + ], + async { + s3.generate_presigned_s3_url( + Url::parse("s3://rattler-s3-testing/my-channel/noarch/repodata.json").unwrap(), + ) + .await + .unwrap() + }, + ) + .await; + assert!( + presigned.to_string() + == "http://localhost:9000/rattler-s3-testing/my-channel/noarch/repodata.json?x-id=GetObject", + "Unexpected presigned URL: {presigned:?}" + ); + } +} diff --git a/crates/rattler_networking/tests/integration_test.rs b/crates/rattler_networking/tests/integration_test.rs new file mode 100644 index 000000000..78a81d54a --- /dev/null +++ b/crates/rattler_networking/tests/integration_test.rs @@ -0,0 +1,341 @@ +use std::{collections::HashMap, path::PathBuf, sync::Arc}; + +use rattler_networking::{ + authentication_storage::backends::file::FileStorage, s3_middleware::S3Config, + AuthenticationMiddleware, AuthenticationStorage, S3Middleware, +}; +use reqwest::Client; +use rstest::*; +use temp_env::async_with_vars; +use tempfile::{tempdir, TempDir}; +use url::Url; + +/* -------------------------------------- UTILS ------------------------------------- */ + +fn run_subprocess(cmd: &str, args: &[&str], env: &HashMap<&str, &str>) -> std::process::Output { + let mut command = std::process::Command::new(cmd); + command.args(args); + for (key, value) in env { + command.env(key, value); + } + let output = command + .output() + .unwrap_or_else(|_| panic!("Failed to execute command {command:?}")); + if !output.status.success() { + eprintln!("Command failed: {command:?}"); + eprintln!("Output: {:?}", String::from_utf8_lossy(&output.stdout)); + eprintln!("Error: {:?}", String::from_utf8_lossy(&output.stderr)); + } + output +} + +/* ------------------------------------ FIXTURES ------------------------------------ */ + +#[fixture] +#[once] +fn minio_host() -> String { + format!( + "http://localhost:{}", + option_env!("MINIO_PORT").unwrap_or("9000") + ) +} + +#[fixture] +#[once] +fn init_channel() { + let host = format!( + "http://minioadmin:minioadmin@localhost:{}", + option_env!("MINIO_PORT").unwrap_or("9000") + ); + let env = &HashMap::from([("MC_HOST_local", host.as_str())]); + let mc_executable = "mc"; + for bucket in &[ + "local/rattler-s3-testing", + "local/rattler-s3-testing-public", + ] { + run_subprocess(mc_executable, &["mb", "--ignore-existing", bucket], env); + run_subprocess( + mc_executable, + &[ + "cp", + PathBuf::from("../../test-data/test-server/repo/noarch/repodata.json") + .to_str() + .unwrap(), + format!("{bucket}/my-channel/noarch/repodata.json").as_str(), + ], + env, + ); + run_subprocess( + mc_executable, + &[ + "cp", + PathBuf::from("../../test-data/test-server/repo/noarch/test-package-0.1-0.tar.bz2") + .to_str() + .unwrap(), + format!("{bucket}/my-channel/noarch/test-package-0.1-0.tar.bz2").as_str(), + ], + env, + ); + } + // Make bucket public + run_subprocess( + mc_executable, + &[ + "anonymous", + "set", + "download", + "local/rattler-s3-testing-public", + ], + env, + ); +} + +#[fixture] +fn aws_config(minio_host: &str) -> (TempDir, std::path::PathBuf) { + let temp_dir = tempdir().unwrap(); + let aws_config = format!( + r#" +[profile default] +aws_access_key_id = minioadmin +aws_secret_access_key = minioadmin +endpoint_url = {minio_host} +region = eu-central-1 + +[profile public] +endpoint_url = {minio_host} +region = eu-central-1 +"# + ); + let aws_config_path = temp_dir.path().join("aws.config"); + std::fs::write(&aws_config_path, aws_config).unwrap(); + (temp_dir, aws_config_path) +} + +/* -------------------------------------- TESTS ------------------------------------- */ + +#[rstest] +#[tokio::test] +async fn test_minio_download_repodata( + minio_host: &str, + #[allow(unused_variables)] init_channel: (), +) { + let temp_dir = tempdir().unwrap(); + let credentials = r#" +{ + "s3://rattler-s3-testing/my-channel": { + "S3Credentials": { + "access_key_id": "minioadmin", + "secret_access_key": "minioadmin" + } + } +}"#; + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, credentials).unwrap(); + let mut auth_storage = AuthenticationStorage::empty(); + auth_storage.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse(minio_host).unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); +} + +#[rstest] +#[tokio::test] +async fn test_minio_download_repodata_public( + minio_host: &str, + #[allow(unused_variables)] init_channel: (), +) { + let auth_storage = AuthenticationStorage::empty(); + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse(minio_host).unwrap(), + region: "eu-central-1".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing-public/my-channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); +} + +#[rstest] +#[tokio::test] +async fn test_minio_download_repodata_aws_profile( + aws_config: (TempDir, std::path::PathBuf), + #[allow(unused_variables)] init_channel: (), +) { + let middleware = S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::empty()); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + AuthenticationStorage::empty(), + ))) + .with(middleware) + .build(); + + let result = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("default")), + ], + async { + download_client + .get("s3://rattler-s3-testing/my-channel/noarch/repodata.json") + .send() + .await + .unwrap() + }, + ) + .await; + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); +} + +#[rstest] +#[tokio::test] +async fn test_minio_download_aws_profile_public( + aws_config: (TempDir, std::path::PathBuf), + #[allow(unused_variables)] init_channel: (), +) { + let middleware = S3Middleware::new(S3Config::FromAWS, AuthenticationStorage::empty()); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new( + AuthenticationMiddleware::from_env_and_defaults().unwrap(), + )) + .with(middleware) + .build(); + let result = async_with_vars( + [ + ("AWS_CONFIG_FILE", Some(aws_config.1.to_str().unwrap())), + ("AWS_PROFILE", Some("public")), + ], + async { + download_client + .get("s3://rattler-s3-testing-public/my-channel/noarch/repodata.json") + .send() + .await + .unwrap() + }, + ) + .await; + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!(body.contains("test-package-0.1-0.tar.bz2")); +} + +#[rstest] +#[tokio::test] +async fn test_cloudflare_r2_download_repodata() { + let temp_dir = tempdir().unwrap(); + + let r2_access_key_id = std::env::var("RATTLER_TEST_R2_ACCESS_KEY_ID").ok(); + let r2_secret_access_key = std::env::var("RATTLER_TEST_R2_SECRET_ACCESS_KEY").ok(); + if r2_access_key_id.is_none() + || r2_access_key_id.clone().unwrap().is_empty() + || r2_secret_access_key.is_none() + || r2_secret_access_key.clone().unwrap().is_empty() + { + eprintln!( + "Skipping test as RATTLER_TEST_R2_ACCESS_KEY_ID or RATTLER_TEST_R2_SECRET_ACCESS_KEY is not set" + ); + return; + } + + let r2_access_key_id = r2_access_key_id.unwrap(); + let r2_secret_access_key = r2_secret_access_key.unwrap(); + + let credentials = format!( + r#" + {{ + "s3://rattler-s3-testing/channel": {{ + "S3Credentials": {{ + "access_key_id": "{}", + "secret_access_key": "{}" + }} + }} + }} + "#, + r2_access_key_id, r2_secret_access_key + ); + + let credentials_path = temp_dir.path().join("credentials.json"); + std::fs::write(&credentials_path, credentials).unwrap(); + + let mut auth_storage = AuthenticationStorage::empty(); + auth_storage.add_backend(Arc::from(FileStorage::from_path(credentials_path).unwrap())); + let middleware = S3Middleware::new( + S3Config::Custom { + endpoint_url: Url::parse( + "https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com", + ) + .unwrap(), + region: "auto".into(), + force_path_style: true, + }, + auth_storage.clone(), + ); + + let download_client = Client::builder().no_gzip().build().unwrap(); + let download_client = reqwest_middleware::ClientBuilder::new(download_client) + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage, + ))) + .with(middleware) + .build(); + + let result = download_client + .get("s3://rattler-s3-testing/channel/noarch/repodata.json") + .send() + .await + .unwrap(); + + assert_eq!(result.status(), 200); + let body = result.text().await.unwrap(); + assert!( + body.contains("my-webserver-0.1.0-pyh4616a5c_0.conda"), + "body does not contain package: {}", + body + ); +} diff --git a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs index 8a101659c..85a9a3e18 100644 --- a/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs +++ b/crates/rattler_repodata_gateway/src/gateway/subdir_builder.rs @@ -54,6 +54,7 @@ impl<'g> SubdirBuilder<'g> { || url.scheme() == "https" || url.scheme() == "gcs" || url.scheme() == "oci" + || url.scheme() == "s3" { let source_config = self.gateway.channel_config.get(&self.channel.base_url); diff --git a/pixi.toml b/pixi.toml index 33691d02b..7bbfe3709 100644 --- a/pixi.toml +++ b/pixi.toml @@ -24,8 +24,10 @@ cxx-compiler = "~=1.6.0" openssl = "~=3.1" make = "~=4.3" pkg-config = "~=0.29.2" -rust = "~=1.80.0" +rust = "~=1.81.0" cmake = "~=3.26.4" +minio-server = ">=2024.12.18" +minio-client = ">=2024.11.21" [target.linux-64.dependencies] clang = ">=18.1.8,<19.0" @@ -45,4 +47,6 @@ pre-commit-install = "pre-commit install" pre-commit-run = "pre-commit run" [environments] -lint = { features = ["lint"], no-default-feature = true, solve-group = "default" } +lint = { features = [ + "lint", +], no-default-feature = true, solve-group = "default" } diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index ed0f80124..75590bebd 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -77,11 +77,61 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arbitrary" @@ -271,9 +321,9 @@ checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", @@ -292,6 +342,381 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "aws-config" +version = "1.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f40e82e858e02445402906e454a73e244c7f501fcae198977585946c48e8697" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.69.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a88f1c30e4ffa2464f910297c24736ff68cca9e8d2b7d52596b54efd99b9c1e" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921a13ed6aabe2d1258f65ef7804946255c799224440774c30e1a2c65cdf983a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196c952738b05dfc917d82a3e9b5ba850822a6d6a86d677afda2a156cc172ceb" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.55.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ef5b73a927ed80b44096f8c20fb4abae65469af15198367e179ae267256e9d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.2.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f45a1c384d7a393026bc5f5c177105aa9fa68e4749653b985707ac27d77295" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "crc64fast-nvme", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper 0.14.32", + "hyper-rustls 0.24.2", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.2.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.2.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -307,6 +732,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.21.7" @@ -319,6 +750,22 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bitflags" version = "2.6.0" @@ -401,6 +848,16 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "bzip2" version = "0.4.4" @@ -447,6 +904,25 @@ dependencies = [ "cipher", ] +[[package]] +name = "cbindgen" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fce8dd7fcfcbf3a0a87d8f515194b49d6135acab73e18bd380d1d93bb1a15eb" +dependencies = [ + "clap", + "heck 0.4.1", + "indexmap 2.7.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.1.34" @@ -495,6 +971,39 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -517,6 +1026,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -552,6 +1067,30 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -561,6 +1100,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crc64fast-nvme" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5e2ee08013e3f228d6d2394116c4549a6df77708442c62d887d83f68ef2ee37" +dependencies = [ + "cbindgen", + "crc", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -586,6 +1135,28 @@ version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -674,6 +1245,16 @@ dependencies = [ "sha2", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -738,12 +1319,44 @@ dependencies = [ "syn", ] +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "elsa" version = "1.10.0" @@ -847,9 +1460,9 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.3.1" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" dependencies = [ "concurrent-queue", "parking", @@ -872,11 +1485,21 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "file_url" -version = "0.2.1" +version = "0.2.2" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "percent-encoding", "thiserror 2.0.9", "typed-path", @@ -897,9 +1520,9 @@ dependencies = [ [[package]] name = "fixedbitset" -version = "0.4.2" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flate2" @@ -926,6 +1549,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "foreign-types" version = "0.3.2" @@ -972,16 +1601,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fslock" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04412b8935272e3a9bae6f48c7bfff74c2911f60525404edfdd28e49884c3bfb" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "funty" version = "2.0.0" @@ -1131,9 +1750,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "google-cloud-auth" @@ -1177,6 +1796,36 @@ dependencies = [ "async-trait", ] +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.6" @@ -1188,7 +1837,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.2.0", "indexmap 2.7.0", "slab", "tokio", @@ -1227,6 +1876,11 @@ name = "hashbrown" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1288,6 +1942,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.2.0" @@ -1299,6 +1964,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1306,7 +1982,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1317,8 +1993,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1328,7 +2004,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92baf25cf0b8c9246baecf3a444546360a97b569168fdf92563ee6a47829920c" dependencies = [ - "http", + "http 1.2.0", "http-serde", "reqwest", "serde", @@ -1341,7 +2017,7 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" dependencies = [ - "http", + "http 1.2.0", "serde", ] @@ -1351,6 +2027,12 @@ version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humansize" version = "2.1.3" @@ -1366,6 +2048,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.5.0" @@ -1375,9 +2081,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.6", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa", "pin-project-lite", @@ -1386,6 +2092,22 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.3" @@ -1393,14 +2115,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.2.0", + "hyper 1.5.0", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.16", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "tower-service", "webpki-roots", ] @@ -1413,7 +2135,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.5.0", "hyper-util", "native-tls", "tokio", @@ -1430,9 +2152,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.5.0", "pin-project-lite", "socket2", "tokio", @@ -1671,6 +2393,12 @@ version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -1689,6 +2417,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1768,9 +2505,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.3.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8e41c97e6bc7ecb552016274b99fbb5d035e8de288c582d9b933af6677bfda" +checksum = "60c7310b93682b36b98fa7ea4de998d3463ccbebd94d935d6b48ba5b6ffa7126" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -1779,9 +2516,9 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.3.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e1d8b05d672c53cb9c7b920bbba8783845ae4f0b076e02a3db1d02c81b4163" +checksum = "4ba01db5ef81e17eb10a5e0f2109d1b3a3e29bac3070fdbd7d156bf7dbd206a1" dependencies = [ "proc-macro2", "quote", @@ -1865,6 +2602,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.0", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2192,6 +2938,23 @@ dependencies = [ "syn", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking" version = "2.2.1" @@ -2252,9 +3015,9 @@ dependencies = [ [[package]] name = "pep508_rs" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c2feee999fa547bacab06a4881bacc74688858b92fa8ef1e206c748b0a76048" +checksum = "faee7227064121fcadcd2ff788ea26f0d8f2bd23a0574da11eca23bc935bcc05" dependencies = [ "boxcar", "indexmap 2.7.0", @@ -2280,9 +3043,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", "indexmap 2.7.0", @@ -2375,6 +3138,16 @@ dependencies = [ "futures-io", ] +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.31" @@ -2463,9 +3236,9 @@ dependencies = [ [[package]] name = "purl" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c14fe28c8495f7eaf77a6e6106966f95211c0a2404b9da50d248fc32af3a3f14" +checksum = "f112b0e2a9bca03924c39166775b74fec9a831f7d4d8fa539dee0e565f403a0e" dependencies = [ "hex", "percent-encoding", @@ -2615,7 +3388,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.23.16", "socket2", "thiserror 1.0.67", "tokio", @@ -2632,7 +3405,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls", + "rustls 0.23.16", "slab", "thiserror 1.0.67", "tinyvec", @@ -2655,9 +3428,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2700,7 +3473,7 @@ dependencies = [ [[package]] name = "rattler" -version = "0.28.8" +version = "0.28.12" dependencies = [ "anyhow", "console", @@ -2711,7 +3484,7 @@ dependencies = [ "humantime", "indexmap 2.7.0", "indicatif", - "itertools 0.13.0", + "itertools 0.14.0", "memchr", "memmap2", "once_cell", @@ -2739,7 +3512,7 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.3.0" +version = "0.3.4" dependencies = [ "anyhow", "dashmap", @@ -2749,7 +3522,7 @@ dependencies = [ "fs4", "futures", "fxhash", - "itertools 0.13.0", + "itertools 0.14.0", "parking_lot", "rattler_conda_types", "rattler_digest", @@ -2767,7 +3540,7 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.29.6" +version = "0.29.10" dependencies = [ "chrono", "dirs", @@ -2777,7 +3550,7 @@ dependencies = [ "glob", "hex", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "lazy-regex", "nom", "purl", @@ -2802,7 +3575,7 @@ dependencies = [ [[package]] name = "rattler_digest" -version = "1.0.4" +version = "1.0.5" dependencies = [ "blake2", "digest", @@ -2817,7 +3590,7 @@ dependencies = [ [[package]] name = "rattler_index" -version = "0.20.3" +version = "0.20.7" dependencies = [ "fs-err", "rattler_conda_types", @@ -2830,13 +3603,13 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.22.35" +version = "0.22.39" dependencies = [ "chrono", "file_url", "fxhash", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "pep440_rs", "pep508_rs", "rattler_conda_types", @@ -2853,7 +3626,7 @@ dependencies = [ [[package]] name = "rattler_macros" -version = "1.0.4" +version = "1.0.5" dependencies = [ "quote", "syn", @@ -2861,19 +3634,21 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.21.9" +version = "0.21.10" dependencies = [ "anyhow", + "async-fd-lock", "async-trait", + "aws-config", + "aws-sdk-s3", "base64 0.22.1", "chrono", "dirs", - "fslock", "getrandom", "google-cloud-auth", "google-cloud-token", - "http", - "itertools 0.13.0", + "http 1.2.0", + "itertools 0.14.0", "keyring", "netrc-rs", "reqwest", @@ -2888,7 +3663,7 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.22.19" +version = "0.22.23" dependencies = [ "bzip2 0.5.0", "chrono", @@ -2915,7 +3690,7 @@ dependencies = [ [[package]] name = "rattler_redaction" -version = "0.1.5" +version = "0.1.6" dependencies = [ "reqwest", "reqwest-middleware", @@ -2924,7 +3699,7 @@ dependencies = [ [[package]] name = "rattler_repodata_gateway" -version = "0.21.28" +version = "0.21.32" dependencies = [ "anyhow", "async-compression", @@ -2940,11 +3715,11 @@ dependencies = [ "fs-err", "futures", "hex", - "http", + "http 1.2.0", "http-cache-semantics", "humansize", "humantime", - "itertools 0.13.0", + "itertools 0.14.0", "json-patch", "libc", "md-5", @@ -2959,6 +3734,7 @@ dependencies = [ "rattler_redaction", "reqwest", "reqwest-middleware", + "retry-policies", "rmp-serde", "serde", "serde_json", @@ -2977,12 +3753,12 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.22.11" +version = "0.22.15" dependencies = [ "enum_dispatch", "fs-err", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "rattler_conda_types", "serde_json", "shlex", @@ -2993,11 +3769,11 @@ dependencies = [ [[package]] name = "rattler_solve" -version = "1.3.0" +version = "1.3.4" dependencies = [ "chrono", "futures", - "itertools 0.13.0", + "itertools 0.14.0", "rattler_conda_types", "rattler_digest", "resolvo", @@ -3009,7 +3785,7 @@ dependencies = [ [[package]] name = "rattler_virtual_packages" -version = "1.1.14" +version = "1.2.0" dependencies = [ "archspec", "libloading", @@ -3117,6 +3893,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.5" @@ -3125,9 +3907,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "async-compression", "base64 0.22.1", @@ -3135,12 +3917,12 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.6", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.5.0", + "hyper-rustls 0.27.3", "hyper-tls", "hyper-util", "ipnet", @@ -3152,9 +3934,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.16", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -3162,8 +3944,9 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.0", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -3182,7 +3965,7 @@ checksum = "d1ccd3b55e711f91a9885a2fa6fbbb2e39db1776420b062efc058c6410f7e5e3" dependencies = [ "anyhow", "async-trait", - "http", + "http 1.2.0", "reqwest", "serde", "thiserror 1.0.67", @@ -3191,9 +3974,9 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03fdd3aa47ae0816ce4ec203eba1330e7c96a6760cbfbee5f1d2ca6e768b50f7" +checksum = "5314eb4b865d39acd1b3cd05eb91b87031bb49fd1278a1bdf8d6680f1389ec29" dependencies = [ "ahash", "bitvec", @@ -3201,7 +3984,7 @@ dependencies = [ "event-listener", "futures", "indexmap 2.7.0", - "itertools 0.13.0", + "itertools 0.14.0", "petgraph", "tracing", ] @@ -3215,6 +3998,17 @@ dependencies = [ "rand", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.17.8" @@ -3264,6 +4058,15 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "0.38.42" @@ -3277,6 +4080,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.16" @@ -3286,11 +4101,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -3303,6 +4130,15 @@ dependencies = [ "security-framework 3.0.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3318,6 +4154,16 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustls-webpki" version = "0.102.8" @@ -3365,6 +4211,30 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "secret-service" version = "4.0.0" @@ -3420,11 +4290,17 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" + [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -3452,9 +4328,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -3485,6 +4361,15 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -3499,9 +4384,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" +checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" dependencies = [ "base64 0.22.1", "chrono", @@ -3517,9 +4402,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" +checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" dependencies = [ "darling", "proc-macro2", @@ -3577,6 +4462,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "simd-adler32" version = "0.3.7" @@ -3674,6 +4569,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3728,9 +4633,9 @@ checksum = "ab16ced94dbd8a46c82fd81e3ed9a8727dac2977ea869d217bcc4ea1f122e81f" [[package]] name = "syn" -version = "2.0.90" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -3796,12 +4701,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3941,13 +4847,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -3965,11 +4881,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + [[package]] name = "toml_datetime" version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] [[package]] name = "toml_edit" @@ -3978,10 +4909,33 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ "indexmap 2.7.0", + "serde", + "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4126,6 +5080,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.11.0" @@ -4169,6 +5129,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "walkdir" version = "2.5.0" @@ -4617,6 +5583,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yansi" version = "1.0.1" diff --git a/py-rattler/Cargo.toml b/py-rattler/Cargo.toml index d88d20878..4a5a929e4 100644 --- a/py-rattler/Cargo.toml +++ b/py-rattler/Cargo.toml @@ -38,6 +38,7 @@ rattler_conda_types = { path = "../crates/rattler_conda_types", default-features rattler_digest = { path = "../crates/rattler_digest" } rattler_networking = { path = "../crates/rattler_networking", default-features = false, features = [ "gcs", + "s3", ] } rattler_shell = { path = "../crates/rattler_shell", default-features = false } rattler_virtual_packages = { path = "../crates/rattler_virtual_packages", default-features = false } diff --git a/py-rattler/rattler/networking/__init__.py b/py-rattler/rattler/networking/__init__.py index dd115384b..d669b7f67 100644 --- a/py-rattler/rattler/networking/__init__.py +++ b/py-rattler/rattler/networking/__init__.py @@ -1,5 +1,17 @@ from rattler.networking.client import Client -from rattler.networking.middleware import MirrorMiddleware, AuthenticationMiddleware, GCSMiddleware from rattler.networking.fetch_repo_data import fetch_repo_data +from rattler.networking.middleware import ( + AuthenticationMiddleware, + GCSMiddleware, + MirrorMiddleware, + S3Middleware, +) -__all__ = ["fetch_repo_data", "Client", "MirrorMiddleware", "AuthenticationMiddleware", "GCSMiddleware"] +__all__ = [ + "fetch_repo_data", + "Client", + "MirrorMiddleware", + "AuthenticationMiddleware", + "GCSMiddleware", + "S3Middleware", +] diff --git a/py-rattler/rattler/networking/client.py b/py-rattler/rattler/networking/client.py index 8aa361013..82cfce37c 100644 --- a/py-rattler/rattler/networking/client.py +++ b/py-rattler/rattler/networking/client.py @@ -1,6 +1,13 @@ from __future__ import annotations + +from rattler.networking.middleware import ( + AuthenticationMiddleware, + GCSMiddleware, + MirrorMiddleware, + OciMiddleware, + S3Middleware, +) from rattler.rattler import PyClientWithMiddleware -from rattler.networking.middleware import AuthenticationMiddleware, MirrorMiddleware, OciMiddleware, GCSMiddleware class Client: @@ -10,7 +17,9 @@ class Client: def __init__( self, - middlewares: list[AuthenticationMiddleware | MirrorMiddleware | OciMiddleware | GCSMiddleware] | None = None, + middlewares: ( + list[AuthenticationMiddleware | MirrorMiddleware | OciMiddleware | GCSMiddleware | S3Middleware] | None + ) = None, ) -> None: self._client = PyClientWithMiddleware( [middleware._middleware for middleware in middlewares] if middlewares else None diff --git a/py-rattler/rattler/networking/middleware.py b/py-rattler/rattler/networking/middleware.py index cbe652f09..49197ea5f 100644 --- a/py-rattler/rattler/networking/middleware.py +++ b/py-rattler/rattler/networking/middleware.py @@ -1,5 +1,13 @@ from __future__ import annotations -from rattler.rattler import PyMirrorMiddleware, PyAuthenticationMiddleware, PyOciMiddleware, PyGCSMiddleware + +from rattler.rattler import ( + PyAuthenticationMiddleware, + PyGCSMiddleware, + PyMirrorMiddleware, + PyOciMiddleware, + PyS3Middleware, + PyS3Config, +) class MirrorMiddleware: @@ -107,6 +115,8 @@ class GCSMiddleware: GCSMiddleware() >>> Client([middleware]) Client() + >>> + ``` """ def __init__(self) -> None: @@ -114,3 +124,68 @@ def __init__(self) -> None: def __repr__(self) -> str: return f"{type(self).__name__}()" + + +class S3Config: + """ + Middleware to work with s3:// URLs + + Examples + -------- + ```python + >>> from rattler.networking import S3Middleware + >>> config = S3Config("http://localhost:9000", "eu-central-1", True) + >>> config + S3Config(http://localhost:9000, eu-central-1, True) + >>> middleware = S3Middleware(config) + >>> middleware + S3Middleware() + >>> S3Config() + S3Config(aws sdk) + >>> + ``` + """ + + def __init__( + self, endpoint_url: str | None = None, region: str | None = None, force_path_style: bool | None = None + ) -> None: + self._config = PyS3Config(endpoint_url, region, force_path_style) + if (endpoint_url is None) != (region is None) or (endpoint_url is None) != (force_path_style is None): + raise ValueError("Invalid arguments for S3Config") + self._endpoint_url = endpoint_url + self._region = region + self._force_path_style = force_path_style + + def __repr__(self) -> str: + inner = ( + f"{self._endpoint_url}, {self._region}, {self._force_path_style}" + if self._endpoint_url is not None + else "aws sdk" + ) + return f"{type(self).__name__}({inner})" + + +class S3Middleware: + """ + Middleware to work with s3:// URLs + + Examples + -------- + ```python + >>> from rattler.networking import Client + >>> middleware = S3Middleware() + >>> middleware + S3Middleware() + >>> Client([middleware]) + Client() + >>> + ``` + """ + + def __init__(self, config: S3Config | None = None) -> None: + if config is None: + config = S3Config() + self._middleware = PyS3Middleware(config._config) + + def __repr__(self) -> str: + return f"{type(self).__name__}()" diff --git a/py-rattler/src/lib.rs b/py-rattler/src/lib.rs index 904d2341a..c88ef17b4 100644 --- a/py-rattler/src/lib.rs +++ b/py-rattler/src/lib.rs @@ -52,7 +52,8 @@ use match_spec::PyMatchSpec; use meta::get_rattler_version; use nameless_match_spec::PyNamelessMatchSpec; use networking::middleware::{ - PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, + PyAuthenticationMiddleware, PyGCSMiddleware, PyMirrorMiddleware, PyOciMiddleware, PyS3Config, + PyS3Middleware, }; use networking::{client::PyClientWithMiddleware, py_fetch_repo_data}; use no_arch_type::PyNoArchType; @@ -107,6 +108,8 @@ fn rattler<'py>(py: Python<'py>, m: Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; // Shell activation things diff --git a/py-rattler/src/networking/client.rs b/py-rattler/src/networking/client.rs index b6cc5a231..6c735b46a 100644 --- a/py-rattler/src/networking/client.rs +++ b/py-rattler/src/networking/client.rs @@ -1,7 +1,8 @@ use crate::{error::PyRattlerError, networking::middleware::PyMiddleware}; use pyo3::{pyclass, pymethods, PyResult}; use rattler_networking::{ - AuthenticationMiddleware, GCSMiddleware, MirrorMiddleware, OciMiddleware, + AuthenticationMiddleware, AuthenticationStorage, GCSMiddleware, MirrorMiddleware, + OciMiddleware, S3Middleware, }; use reqwest_middleware::ClientWithMiddleware; @@ -36,6 +37,13 @@ impl PyClientWithMiddleware { PyMiddleware::Gcs(middleware) => { client = client.with(GCSMiddleware::from(middleware)); } + PyMiddleware::S3(middleware) => { + client = client.with(S3Middleware::new( + middleware.s3_config.into(), + AuthenticationStorage::from_env_and_defaults() + .map_err(PyRattlerError::from)?, + )); + } } } let client = client.build(); diff --git a/py-rattler/src/networking/middleware.rs b/py-rattler/src/networking/middleware.rs index 3e01bbf9a..270b32860 100644 --- a/py-rattler/src/networking/middleware.rs +++ b/py-rattler/src/networking/middleware.rs @@ -1,6 +1,7 @@ use pyo3::{pyclass, pymethods, FromPyObject, PyResult}; use rattler_networking::{ - mirror_middleware::Mirror, GCSMiddleware, MirrorMiddleware, OciMiddleware, + mirror_middleware::Mirror, s3_middleware::S3Config, GCSMiddleware, MirrorMiddleware, + OciMiddleware, }; use std::collections::HashMap; use url::Url; @@ -13,6 +14,7 @@ pub enum PyMiddleware { Authentication(PyAuthenticationMiddleware), Oci(PyOciMiddleware), Gcs(PyGCSMiddleware), + S3(PyS3Middleware), } #[pyclass] @@ -106,3 +108,69 @@ impl From for GCSMiddleware { GCSMiddleware } } + +#[derive(Clone)] +#[pyclass] +pub struct PyS3Config { + // non-trivial enums are not supported by pyo3 as pyclasses + pub(crate) custom: Option, +} + +#[derive(Clone)] +pub(crate) struct PyS3ConfigCustom { + pub(crate) endpoint_url: Url, + pub(crate) region: String, + pub(crate) force_path_style: bool, +} + +#[pymethods] +impl PyS3Config { + #[new] + #[pyo3(signature = (endpoint_url=None, region=None, force_path_style=None))] + pub fn __init__( + endpoint_url: Option, + region: Option, + force_path_style: Option, + ) -> PyResult { + match (endpoint_url, region, force_path_style) { + (Some(endpoint_url), Some(region), Some(force_path_style)) => Ok(Self { + custom: Some(PyS3ConfigCustom { + endpoint_url: Url::parse(&endpoint_url) + .map_err(PyRattlerError::from) + .unwrap(), + region, + force_path_style, + }), + }), + (None, None, None) => Ok(Self { custom: None }), + _ => unreachable!("Case handled in python"), + } + } +} + +impl From for S3Config { + fn from(_value: PyS3Config) -> Self { + match _value.custom { + None => S3Config::FromAWS, + Some(custom) => S3Config::Custom { + endpoint_url: custom.endpoint_url, + region: custom.region, + force_path_style: custom.force_path_style, + }, + } + } +} + +#[pyclass] +#[derive(Clone)] +pub struct PyS3Middleware { + pub(crate) s3_config: PyS3Config, +} + +#[pymethods] +impl PyS3Middleware { + #[new] + pub fn __init__(s3_config: PyS3Config) -> PyResult { + Ok(Self { s3_config }) + } +}