Replace localstack with minio

.env.example

@@ -16,14 +16,13 @@ KEEP_CONTAINERS_UP="FALSE"
 # e.g. "*"
 LUMI_API_CORS_ALLOWED_ORIGINS=${LUMI_API_CORS_ALLOWED_ORIGINS:-http://localhost,http://localhost:3000}
 # AWS Variables for S3 Object Storage
-# Configure these for AWS access, or use defaults for local development with LocalStack.
-AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-test}
-AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-test}
+# Configure these for AWS access, or use defaults for local development with minio.
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-lumigator}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-lumigator}
 AWS_DEFAULT_REGION=${AWS_DEFAULT_REGION:-us-east-2}
-AWS_ENDPOINT_URL=${AWS_ENDPOINT_URL:-http://localhost:4566}
+# Default is the default api port used by minio
+AWS_ENDPOINT_URL=${AWS_ENDPOINT_URL:-http://localhost:9000}
 S3_BUCKET=${S3_BUCKET:-lumigator-storage}
-# Optional: Required only if using LocalStack with advanced features/LocalStack Pro.
-LOCALSTACK_AUTH_TOKEN=<your_localstack_auth_token>
 # Ray Cluster Configuration
 # These settings are for the local Ray setup. To use an external Ray cluster, you MUST use an external S3-compatible storage
 # to ensure the Ray workers can access data from your Lumigator server.

docker-compose.yaml

@@ -2,34 +2,41 @@ name: lumigator
 
 services:
 
-  localstack:
-    # if you want to enable local S3-style data persistence, use the following
-    # image and set LOCALSTACK_AUTH_TOKEN in your env
-    # image: localstack/localstack-pro:3.4.0
-    image: localstack/localstack:3.4.0
-    platform: linux/amd64
+  minio:
+    image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z
+    command: server /data --console-address ":9001"
     ports:
-      - 4566:4566
+      - 9000:9000
+      - 9001:9001
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
     environment:
-      - SERVICES=s3:4566
-      - CREATE_BUCKETS=lumigator-storage
-      - LOCALSTACK_AUTH_TOKEN
-      - PERSISTENCE=1
-      - SNAPSHOT_SAVE_STRATEGY=ON_REQUEST
-      - EXTRA_CORS_ALLOWED_ORIGINS=*
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+      - MINIO_API_CORS_ALLOW_ORIGIN=*
     volumes:
-      - localstack-data:/var/lib/localstack
+    # - ${HOME}/minio/data:/data
+      - minio-data:/data
     profiles:
       - local
 
-  localstack-create-bucket:
-    image: localstack/localstack:3.4.0
-    platform: linux/amd64
+  minio-admin:
+    image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z
     depends_on:
-      localstack:
+      minio:
         condition: service_healthy
-    entrypoint: >
-       bash -c "awslocal s3 mb s3://lumigator-storage"
+    entrypoint:
+      - /bin/bash
+      - -c
+      - |
+          set -ex
+          mc alias set lumigator_s3 http://minio:9000 minioadmin minioadmin
+          mc admin user add lumigator_s3 lumigator lumigator
+          mc admin policy attach lumigator_s3 readwrite --user lumigator
+          mc mb -p lumigator_s3/lumigator-storage
     extra_hosts:
       - "localhost:host-gateway"
     profiles:
@@ -87,7 +94,9 @@ services:
       target: "main_image"
     platform: linux/amd64
     depends_on:
-      localstack:
+      minio-admin:
+        condition: service_completed_successfully
+      minio:
         condition: "service_started"
         required: false
       ray:
@@ -147,5 +156,5 @@ services:
 
 volumes:
 
-    localstack-data:
+    minio-data:
     database_volume:

lumigator/python/mzai/backend/README.md

@@ -18,6 +18,10 @@ uv sync --dev && \
 source .venv/bin/activate
 ```
 
+## Make usage
+
+The available Makefile will copy a `.env` file from the existing `.env.template` file. There are some targets to start the docker composition (`local-up`, `start-lumigator`, `start-lumigator-build`), to stop it (`local-down`) and to run tests (`test-backend`, `test-sdk`, `test-all`).
+
 ## Test instructions
 
 The backend includes both unit tests (requiring no additional containers) and integration tests (currently requiring a live Ray instance in the same network where the application and tests are running, and test LocalStack containers also in the same configuration).

lumigator/python/mzai/backend/backend/tests/conftest.py

@@ -3,7 +3,7 @@
 import os
 import uuid
 from pathlib import Path
-from unittest.mock import Mock, patch
+from unittest.mock import MagicMock, patch
 
 import boto3
 import fsspec
@@ -130,32 +130,32 @@ def fake_s3fs() -> S3FileSystem:
 @pytest.fixture(scope="function")
 def fake_s3_client(fake_s3fs) -> S3Client:
     """Provide a fake S3 client using MemoryFileSystem as underlying storage."""
-    os.environ["AWS_ACCESS_KEY_ID"] = "test"
+    os.environ["AWS_ACCESS_KEY_ID"] = "lumigator"
     # Please check https://github.com/localstack/localstack/issues/5894
     # for info about the test region used
-    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-    os.environ["AWS_SECRET_ACCESS_KEY"] = "test"  # pragma: allowlist secret
-    os.environ["AWS_ENDPOINT_URL"] = "http://example.com:4566"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-2"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "lumigator"  # pragma: allowlist secret
+    os.environ["AWS_ENDPOINT_URL"] = "http://example.com:9000"
     return FakeS3Client(MemoryFileSystem.store)
 
 
 @pytest.fixture(scope="function")
 def boto_s3_client() -> S3Client:
     """Provide a real S3 client."""
-    os.environ["AWS_ACCESS_KEY_ID"] = "test"
+    os.environ["AWS_ACCESS_KEY_ID"] = "lumigator"
     # Please check https://github.com/localstack/localstack/issues/5894
     # for info about the test region used
-    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
-    os.environ["AWS_SECRET_ACCESS_KEY"] = "test"  # pragma: allowlist secret
-    os.environ["AWS_ENDPOINT_URL"] = "http://localhost:4566"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-2"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "lumigator"  # pragma: allowlist secret
+    os.environ["AWS_ENDPOINT_URL"] = "http://localhost:9000"
     return boto3.client("s3")
 
 
 @pytest.fixture(scope="function")
 def boto_s3fs() -> S3FileSystem:
     """Provide a real s3fs client."""
     s3fs = S3FileSystem()
-    mock_s3fs = Mock(wraps=s3fs)
+    mock_s3fs = MagicMock(wraps=s3fs)
     yield mock_s3fs
     logger.info(f"intercepted s3fs calls: {str(mock_s3fs.mock_calls)}")