feat: Add S3 support

.github/workflows/rust-compile.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
     paths:
       # When we change pyproject.toml, we want to ensure that the maturin builds still work
@@ -22,7 +22,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: "-D warnings"
   CARGO_TERM_COLOR: always
-  DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c
+  DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c,s3
 
 jobs:
   check-rustdoc-links:
@@ -54,28 +54,27 @@ jobs:
   build:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
-    needs: [ format_and_lint ]
+    needs: [format_and_lint]
     strategy:
       fail-fast: false
       matrix:
         include:
           - { name: "Linux-x86_64",      target: x86_64-unknown-linux-musl,        os: ubuntu-22.04 }
           - { name: "Linux-aarch64",     target: aarch64-unknown-linux-musl,       os: ubuntu-latest, skip-tests: true }
           - { name: "Linux-arm",         target: arm-unknown-linux-musleabi,       os: ubuntu-latest, use-cross: true, skip-tests: true }
-
           # - { name: "Linux-mips",        target: mips-unknown-linux-musl,          os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mipsel",      target: mipsel-unknown-linux-musl,        os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mips64",      target: mips64-unknown-linux-muslabi64,   os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mips64el",    target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true }
 
-#          - { name: "Linux-powerpc",     target: powerpc-unknown-linux-gnu,        os: ubuntu-latest, use-cross: true, skip-tests: true }
+          # - { name: "Linux-powerpc",     target: powerpc-unknown-linux-gnu,        os: ubuntu-latest, use-cross: true, skip-tests: true }
           - { name: "Linux-powerpc64",   target: powerpc64-unknown-linux-gnu,      os: ubuntu-latest, use-cross: true, skip-tests: true }
           - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu,    os: ubuntu-latest, use-cross: true, skip-tests: true }
 
           - { name: "Linux-s390x",       target: s390x-unknown-linux-gnu,          os: ubuntu-latest, use-cross: true, skip-tests: true }
 
           - { name: "macOS-x86_64",      target: x86_64-apple-darwin,              os: macOS-latest }
-          - { name: "macOS-aarch64",     target: aarch64-apple-darwin,             os: macOS-latest,                  skip-tests: true }
+          - { name: "macOS-aarch64",     target: aarch64-apple-darwin,             os: macOS-latest }
 
           - { name: "Windows-x86_64",    target: x86_64-pc-windows-msvc,           os: windows-latest }
           - { name: "Windows-aarch64",   target: aarch64-pc-windows-msvc,          os: windows-latest,                  skip-tests: true }
@@ -121,11 +120,11 @@ jobs:
 
       - name: Build
         run: >
-          cargo build 
-          --all-targets  
+          cargo build
+          --all-targets
           --features ${{ env.DEFAULT_FEATURES }}
           --target ${{ matrix.target }}
-          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}}
+          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }}
 
       - name: Disable testing the tools crate if cross compiling
         id: test-options
@@ -139,17 +138,33 @@ jobs:
         with:
           tool: cargo-nextest
 
+      - name: Set up pixi
+        if: ${{ !matrix.skip-tests }}
+        uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          run-install: false
+          # on windows, the minio server doesn't get shutdown properly so there is still a lock on this file
+          post-cleanup: false
+
+      - name: Install minio for integration tests
+        if: ${{ !matrix.skip-tests }}
+        run: |
+          pixi global install minio-client
+          pixi global install minio-server
+
       - name: Run tests
         if: ${{ !matrix.skip-tests }}
         env:
           GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }}
-        run: >
-          cargo nextest run
-          --workspace 
-          --features ${{ env.DEFAULT_FEATURES }} 
-          --target ${{ matrix.target }}
-          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} 
-          ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} 
+          RATTLER_TEST_R2_ACCESS_KEY_ID: ${{ secrets.RATTLER_TEST_R2_ACCESS_KEY_ID }}
+          RATTLER_TEST_R2_SECRET_ACCESS_KEY: ${{ secrets.RATTLER_TEST_R2_SECRET_ACCESS_KEY }}
+        run: |
+          minio server --help
+          mkdir -p ${{ runner.temp }}/minio-data
+          minio server --address 127.0.0.1:9000 ${{ runner.temp }}/minio-data &
+          sleep 5
+          curl -I http://localhost:9000/minio/health/live
+          cargo nextest run --workspace --features ${{ env.DEFAULT_FEATURES }} --target ${{ matrix.target }} ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }} ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }}
 
       - name: Run doctests
         if: ${{ !matrix.skip-tests }}

Cargo.toml

@@ -72,6 +72,16 @@ getrandom = { version = "0.2.15", default-features = false }
 glob = "0.3.2"
 google-cloud-auth = { version = "0.17.2", default-features = false }
 google-cloud-token = "0.1.2"
+aws-config = { version = "1.5.14", default-features = false, features = [
+    "rt-tokio",
+    "rustls",
+    "sso",
+] }
+aws-sdk-s3 = { version = "1.69.0", default-features = false, features = [
+    "rt-tokio",
+    "rustls",
+    "sigv4a",
+] }
 hex = "0.4.3"
 hex-literal = "0.4.1"
 http = "1.2"
@@ -142,7 +152,7 @@ sysinfo = "0.33.1"
 tar = "0.4.43"
 tempdir = "0.3.7"
 tempfile = "3.15.0"
-temp-env = "0.3.6"
+temp-env = { version = "0.3.6", features = ["async_closure"] }
 test-log = "0.2.16"
 thiserror = "2.0"
 tokio = { version = "1.42.0", default-features = false }

crates/rattler-bin/Cargo.toml

@@ -29,7 +29,7 @@ indicatif = { workspace = true }
 once_cell = { workspace = true }
 rattler = { path="../rattler", version = "0.28.12", default-features = false, features = ["indicatif"] }
 rattler_conda_types = { path="../rattler_conda_types", version = "0.29.10", default-features = false }
-rattler_networking = { path="../rattler_networking", version = "0.21.10", default-features = false, features = ["gcs"] }
+rattler_networking = { path="../rattler_networking", version = "0.21.10", default-features = false, features = ["gcs", "s3"] }
 rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.32", default-features = false, features = ["gateway"] }
 rattler_solve = { path="../rattler_solve", version = "1.3.4", default-features = false, features = ["resolvo", "libsolv_c"] }
 rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.2.0", default-features = false }

crates/rattler-bin/src/commands/create.rs

@@ -21,7 +21,9 @@ use rattler_conda_types::{
     Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, ParseStrictness, Platform,
     PrefixRecord, RepoDataRecord, Version,
 };
-use rattler_networking::AuthenticationMiddleware;
+use rattler_networking::{
+    s3_middleware::S3Config, AuthenticationMiddleware, AuthenticationStorage,
+};
 use rattler_repodata_gateway::{Gateway, RepoData, SourceConfig};
 use rattler_solve::{
     libsolv_c::{self},
@@ -150,6 +152,10 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
     let download_client = reqwest_middleware::ClientBuilder::new(download_client)
         .with_arc(Arc::new(AuthenticationMiddleware::from_env_and_defaults()?))
         .with(rattler_networking::OciMiddleware)
+        .with(rattler_networking::S3Middleware::new(
+            S3Config::FromAWS,
+            AuthenticationStorage::from_env_and_defaults()?,
+        ))
         .with(rattler_networking::GCSMiddleware)
         .build();
 

crates/rattler/src/cli/auth.rs

@@ -26,6 +26,18 @@ pub struct LoginArgs {
     /// The token to use on anaconda.org / quetz authentication
     #[clap(long)]
     conda_token: Option<String>,
+
+    /// The S3 access key ID
+    #[clap(long, requires_all = ["s3_secret_access_key"], conflicts_with_all = ["token", "username", "password", "conda_token"])]
+    s3_access_key_id: Option<String>,
+
+    /// The S3 secret access key
+    #[clap(long, requires_all = ["s3_access_key_id"])]
+    s3_secret_access_key: Option<String>,
+
+    /// The S3 session token
+    #[clap(long, requires_all = ["s3_access_key_id"])]
+    s3_session_token: Option<String>,
 }
 
 #[derive(Parser, Debug)]
@@ -73,6 +85,10 @@ pub enum AuthenticationCLIError {
     #[error("Authentication with anaconda.org requires a conda token. Use `--conda-token` to provide one")]
     AnacondaOrgBadMethod,
 
+    /// Bad authentication method when using S3
+    #[error("Authentication with S3 requires a S3 access key ID and a secret access key. Use `--s3-access-key-id` and `--s3-secret-access-key` to provide them")]
+    S3BadMethod,
+
     /// Wrapper for errors that are generated from the underlying storage system
     /// (keyring or file system)
     #[error("Failed to initialize the authentication storage system")]
@@ -86,7 +102,7 @@ pub enum AuthenticationCLIError {
 
 fn get_url(url: &str) -> Result<String, AuthenticationCLIError> {
     // parse as url and extract host without scheme or port
-    let host = if url.contains("://") {
+    let host = if url.contains("http://") || url.contains("https://") {
         url::Url::parse(url)?.host_str().unwrap().to_string()
     } else {
         url.to_string()
@@ -117,6 +133,15 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti
         }
     } else if let Some(token) = args.token {
         Authentication::BearerToken(token)
+    } else if let (Some(access_key_id), Some(secret_access_key)) =
+        (args.s3_access_key_id, args.s3_secret_access_key)
+    {
+        let session_token = args.s3_session_token;
+        Authentication::S3Credentials {
+            access_key_id,
+            secret_access_key,
+            session_token,
+        }
     } else {
         return Err(AuthenticationCLIError::NoAuthenticationMethod);
     };
@@ -129,6 +154,14 @@ fn login(args: LoginArgs, storage: AuthenticationStorage) -> Result<(), Authenti
         return Err(AuthenticationCLIError::AnacondaOrgBadMethod);
     }
 
+    if host.starts_with("s3://") && !matches!(auth, Authentication::S3Credentials { .. }) {
+        return Err(AuthenticationCLIError::S3BadMethod);
+    }
+
+    if matches!(auth, Authentication::S3Credentials { .. }) && !host.starts_with("s3://") {
+        return Err(AuthenticationCLIError::S3BadMethod);
+    }
+
     storage
         .store(&host, &auth)
         .map_err(AuthenticationCLIError::StorageError)?;

crates/rattler_networking/Cargo.toml

@@ -15,6 +15,7 @@ default = ["native-tls"]
 native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"]
 rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"]
 gcs = ["google-cloud-auth", "google-cloud-token"]
+s3 = ["aws-config", "aws-sdk-s3"]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -25,9 +26,17 @@ chrono = { workspace = true }
 dirs = { workspace = true }
 google-cloud-auth = { workspace = true, optional = true }
 google-cloud-token = { workspace = true, optional = true }
+aws-config = { workspace = true, optional = true }
+aws-sdk-s3 = { workspace = true, optional = true }
 http = { workspace = true }
 itertools = { workspace = true }
-keyring = { workspace = true, features = ["apple-native", "windows-native", "async-secret-service", "async-io", "crypto-rust"] }
+keyring = { workspace = true, features = [
+    "apple-native",
+    "windows-native",
+    "async-secret-service",
+    "async-io",
+    "crypto-rust",
+] }
 netrc-rs = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 reqwest-middleware = { workspace = true }
@@ -50,3 +59,5 @@ axum = { workspace = true }
 reqwest-retry = { workspace = true }
 sha2 = { workspace = true }
 temp-env = { workspace = true }
+rstest = { workspace = true }
+rand = { workspace = true }

crates/rattler_networking/src/authentication_middleware.rs

@@ -113,7 +113,7 @@ impl AuthenticationMiddleware {
                         .insert(reqwest::header::AUTHORIZATION, header_value);
                     Ok(req)
                 }
-                Authentication::CondaToken(_) => Ok(req),
+                Authentication::CondaToken(_) | Authentication::S3Credentials { .. } => Ok(req),
             }
         } else {
             Ok(req)

crates/rattler_networking/src/authentication_storage/authentication.rs

@@ -19,6 +19,15 @@ pub enum Authentication {
     },
     /// A conda token is sent in the URL as `/t/{TOKEN}/...`
     CondaToken(String),
+    /// S3 credentials
+    S3Credentials {
+        /// The access key ID to use for S3 authentication
+        access_key_id: String,
+        /// The secret access key to use for S3 authentication
+        secret_access_key: String,
+        /// The session token to use for S3 authentication
+        session_token: Option<String>,
+    },
 }
 
 /// An error that can occur when parsing an authentication string

crates/rattler_networking/src/authentication_storage/storage.rs

@@ -142,6 +142,32 @@ impl AuthenticationStorage {
             Ok(Some(credentials)) => return Ok((url, Some(credentials))),
         };
 
+        // S3 protocol URLs need to be treated separately since they follow a different schema
+        if url.scheme() == "s3" {
+            let mut current_url = url.clone();
+            loop {
+                match self.get(current_url.as_str()) {
+                    Ok(None) => {
+                        let possible_rest =
+                            current_url.as_str().rsplit_once('/').map(|(rest, _)| rest);
+
+                        match possible_rest {
+                            Some(rest) => {
+                                if let Ok(new_url) = Url::parse(rest) {
+                                    current_url = new_url;
+                                } else {
+                                    return Ok((url, None));
+                                }
+                            }
+                            _ => return Ok((url, None)), // No more subpaths to check
+                        }
+                    }
+                    Ok(Some(credentials)) => return Ok((url, Some(credentials))),
+                    Err(_) => return Ok((url, None)),
+                }
+            }
+        }
+
         // Check for credentials under e.g. `*.prefix.dev`
         let Some(mut domain) = url.domain() else {
             return Ok((url, None));

crates/rattler_networking/src/lib.rs

@@ -11,6 +11,11 @@ pub mod gcs_middleware;
 #[cfg(feature = "gcs")]
 pub use gcs_middleware::GCSMiddleware;
 
+#[cfg(feature = "s3")]
+pub mod s3_middleware;
+#[cfg(feature = "s3")]
+pub use s3_middleware::S3Middleware;
+
 pub mod authentication_middleware;
 pub mod authentication_storage;