feat: Add S3 support

.github/workflows/rust-compile.yml

@@ -1,6 +1,6 @@
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
     paths:
       # When we change pyproject.toml, we want to ensure that the maturin builds still work
@@ -22,7 +22,7 @@ env:
   RUST_BACKTRACE: 1
   RUSTFLAGS: "-D warnings"
   CARGO_TERM_COLOR: always
-  DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c
+  DEFAULT_FEATURES: indicatif,tokio,serde,wasm,reqwest,sparse,gateway,resolvo,libsolv_c,s3
 
 jobs:
   check-rustdoc-links:
@@ -54,21 +54,20 @@ jobs:
   build:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
-    needs: [ format_and_lint ]
+    needs: [format_and_lint]
     strategy:
       fail-fast: false
       matrix:
         include:
           - { name: "Linux-x86_64",      target: x86_64-unknown-linux-musl,        os: ubuntu-22.04 }
           - { name: "Linux-aarch64",     target: aarch64-unknown-linux-musl,       os: ubuntu-latest, skip-tests: true }
           - { name: "Linux-arm",         target: arm-unknown-linux-musleabi,       os: ubuntu-latest, use-cross: true, skip-tests: true }
-
           # - { name: "Linux-mips",        target: mips-unknown-linux-musl,          os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mipsel",      target: mipsel-unknown-linux-musl,        os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mips64",      target: mips64-unknown-linux-muslabi64,   os: ubuntu-latest, use-cross: true, skip-tests: true }
           # - { name: "Linux-mips64el",    target: mips64el-unknown-linux-muslabi64, os: ubuntu-latest, use-cross: true, skip-tests: true }
 
-#          - { name: "Linux-powerpc",     target: powerpc-unknown-linux-gnu,        os: ubuntu-latest, use-cross: true, skip-tests: true }
+          # - { name: "Linux-powerpc",     target: powerpc-unknown-linux-gnu,        os: ubuntu-latest, use-cross: true, skip-tests: true }
           - { name: "Linux-powerpc64",   target: powerpc64-unknown-linux-gnu,      os: ubuntu-latest, use-cross: true, skip-tests: true }
           - { name: "Linux-powerpc64le", target: powerpc64le-unknown-linux-gnu,    os: ubuntu-latest, use-cross: true, skip-tests: true }
 
@@ -121,11 +120,11 @@ jobs:
 
       - name: Build
         run: >
-          cargo build 
-          --all-targets  
+          cargo build
+          --all-targets
           --features ${{ env.DEFAULT_FEATURES }}
           --target ${{ matrix.target }}
-          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}}
+          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }}
 
       - name: Disable testing the tools crate if cross compiling
         id: test-options
@@ -139,17 +138,27 @@ jobs:
         with:
           tool: cargo-nextest
 
+      - name: Set up pixi
+        uses: prefix-dev/setup-pixi@v0.8.1
+        with:
+          run-install: false
+
+      - name: Install minio
+        run: |
+          pixi global install minio-server
+          pixi global install minio-client
+
       - name: Run tests
         if: ${{ !matrix.skip-tests }}
         env:
           GOOGLE_CLOUD_TEST_KEY_JSON: ${{ secrets.GOOGLE_CLOUD_TEST_KEY_JSON }}
         run: >
           cargo nextest run
-          --workspace 
-          --features ${{ env.DEFAULT_FEATURES }} 
+          --workspace
+          --features ${{ env.DEFAULT_FEATURES }}
           --target ${{ matrix.target }}
-          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS}} 
-          ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} 
+          ${{ steps.build-options.outputs.CARGO_BUILD_OPTIONS }}
+          ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS }}
 
       - name: Run doctests
         if: ${{ !matrix.skip-tests }}

Cargo.toml

@@ -72,6 +72,9 @@ getrandom = { version = "0.2.15", default-features = false }
 glob = "0.3.1"
 google-cloud-auth = { version = "0.17.2", default-features = false }
 google-cloud-token = "0.1.2"
+aws-config = "1.5.12"
+aws-sdk-s3 = "1.67.0"
+aws-runtime = "1.5.2"
 hex = "0.4.3"
 hex-literal = "0.4.1"
 http = "1.2"
@@ -126,6 +129,7 @@ serde-value = "0.7.0"
 serde_with = "3.11.0"
 serde_yaml = "0.9.34"
 serde-untagged = "0.1.6"
+serial_test = "0.4.0"
 sha2 = "0.10.8"
 shlex = "1.3.0"
 similar-asserts = "1.6.0"

crates/rattler-bin/Cargo.toml

@@ -29,7 +29,7 @@ indicatif = { workspace = true }
 once_cell = { workspace = true }
 rattler = { path="../rattler", version = "0.28.9", default-features = false, features = ["indicatif"] }
 rattler_conda_types = { path="../rattler_conda_types", version = "0.29.7", default-features = false }
-rattler_networking = { path="../rattler_networking", version = "0.21.9", default-features = false, features = ["gcs"] }
+rattler_networking = { path="../rattler_networking", version = "0.21.9", default-features = false, features = ["gcs", "s3"] }
 rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.21.29", default-features = false, features = ["gateway"] }
 rattler_solve = { path="../rattler_solve", version = "1.3.1", default-features = false, features = ["resolvo", "libsolv_c"] }
 rattler_virtual_packages = { path="../rattler_virtual_packages", version = "1.1.15", default-features = false }

crates/rattler-bin/src/commands/create.rs

@@ -153,6 +153,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
             authentication_storage,
         )))
         .with(rattler_networking::OciMiddleware)
+        .with(rattler_networking::S3Middleware::new(None, None, None))
         .with(rattler_networking::GCSMiddleware)
         .build();
 

crates/rattler_networking/Cargo.toml

@@ -15,6 +15,7 @@ default = ["native-tls"]
 native-tls = ["reqwest/native-tls", "google-cloud-auth?/default-tls"]
 rustls-tls = ["reqwest/rustls-tls", "google-cloud-auth?/rustls-tls"]
 gcs = ["google-cloud-auth", "google-cloud-token"]
+s3 = ["aws-config", "aws-sdk-s3", "aws-runtime"]
 
 [dependencies]
 anyhow = { workspace = true }
@@ -25,9 +26,18 @@ dirs = { workspace = true }
 fslock = { workspace = true }
 google-cloud-auth = { workspace = true, optional = true }
 google-cloud-token = { workspace = true, optional = true }
+aws-config = { workspace = true, optional = true }
+aws-sdk-s3 = { workspace = true, optional = true }
+aws-runtime = { workspace = true, optional = true }
 http = { workspace = true }
 itertools = { workspace = true }
-keyring = { workspace = true, features = ["apple-native", "windows-native", "async-secret-service", "async-io", "crypto-rust"] }
+keyring = { workspace = true, features = [
+    "apple-native",
+    "windows-native",
+    "async-secret-service",
+    "async-io",
+    "crypto-rust",
+] }
 netrc-rs = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 reqwest-middleware = { workspace = true }
@@ -50,3 +60,6 @@ axum = { workspace = true }
 reqwest-retry = { workspace = true }
 sha2 = { workspace = true }
 temp-env = { workspace = true }
+rstest = { workspace = true }
+rand = { workspace = true }
+serial_test = { workspace = true }

crates/rattler_networking/src/lib.rs

@@ -11,6 +11,11 @@ pub mod gcs_middleware;
 #[cfg(feature = "gcs")]
 pub use gcs_middleware::GCSMiddleware;
 
+#[cfg(feature = "s3")]
+pub mod s3_middleware;
+#[cfg(feature = "s3")]
+pub use s3_middleware::S3Middleware;
+
 pub mod authentication_middleware;
 pub mod authentication_storage;