diff --git a/Dockerfile.setup b/Dockerfile.setup index 409028c..f369163 100644 --- a/Dockerfile.setup +++ b/Dockerfile.setup @@ -1,12 +1,11 @@ -FROM python:3.10-alpine +FROM python:alpine -RUN apk add --no-cache bash openssh ca-certificates jq curl openssl perl git zip \ +RUN apk add --no-cache aws-cli bash openssh ca-certificates jq curl openssl perl git zip \ && apk add --no-cache --virtual .build-deps gcc make openssl-dev libffi-dev musl-dev linux-headers \ && apk add --no-cache libintl icu-libs libc6-compat \ && apk add --no-cache bash-completion \ && update-ca-certificates RUN pip3 install azure-cli -RUN pip3 install awscli RUN az config set extension.use_dynamic_install=yes_without_prompt -ENTRYPOINT ["/bin/bash"] \ No newline at end of file +ENTRYPOINT ["/bin/bash"] diff --git a/bin/localstack-setup_emails.sh b/bin/localstack-setup_emails.sh index 4a0b18a..b20ced5 100644 --- a/bin/localstack-setup_emails.sh +++ b/bin/localstack-setup_emails.sh @@ -34,10 +34,11 @@ wait_for "DynamoDB" "aws dynamodb list-tables --endpoint-url=$ENDPOINT" echo "Create delta-rs lock table in dynamo" aws dynamodb delete-table --table-name locks --endpoint-url=$ENDPOINT > /dev/null 2>&1 -aws dynamodb create-table --table-name locks --endpoint-url=$ENDPOINT \ - --attribute-definitions \ - AttributeName=key,AttributeType=S \ - --key-schema \ - AttributeName=key,KeyType=HASH \ - --provisioned-throughput \ - ReadCapacityUnits=10,WriteCapacityUnits=10 > /dev/null \ No newline at end of file +aws dynamodb create-table \ + --endpoint-url=$ENDPOINT \ + --table-name locks \ + --attribute-definitions AttributeName=tablePath,AttributeType=S \ + AttributeName=fileName,AttributeType=S \ + --key-schema AttributeName=tablePath,KeyType=HASH \ + AttributeName=fileName,KeyType=RANGE \ + --billing-mode PAY_PER_REQUEST diff --git a/tests/emails_s3_tests.rs b/tests/emails_s3_tests.rs index 1c22191..8d5e652 100644 --- a/tests/emails_s3_tests.rs +++ b/tests/emails_s3_tests.rs @@ -31,6 +31,7 @@ async fn when_rebalance_happens() { } async fn run_emails_s3_tests(initiate_rebalance: bool) { + deltalake_aws::register_handlers(None); helpers::init_logger(); let topic = format!("emails_s3-{}", Uuid::new_v4()); let table = prepare_table(&topic).await; @@ -80,7 +81,7 @@ fn create_options(name: &str) -> IngestOptions { env::set_var("AWS_S3_LOCKING_PROVIDER", "dynamodb"); env::set_var("AWS_REGION", "us-east-2"); env::set_var("AWS_STORAGE_ALLOW_HTTP", "true"); - env::set_var("DYNAMO_LOCK_TABLE_NAME", "locks"); + env::set_var("DELTA_DYNAMO_TABLE_NAME", "locks"); env::set_var("DYNAMO_LOCK_OWNER_NAME", name); env::set_var("DYNAMO_LOCK_PARTITION_KEY_VALUE", "emails_s3_tests"); env::set_var("DYNAMO_LOCK_REFRESH_PERIOD_MILLIS", "100"); diff --git a/tests/helpers/mod.rs b/tests/helpers/mod.rs index 116aef7..737b6fc 100644 --- a/tests/helpers/mod.rs +++ b/tests/helpers/mod.rs @@ -213,6 +213,11 @@ pub async fn create_and_run_kdi( Runtime, ) { init_logger(); + + #[cfg(feature = "s3")] + deltalake_aws::register_handlers(None); + #[cfg(feature = "azure")] + deltalake_azure::register_handlers(None); println!("OPTS!: {:?}", opts); let topic = format!("{}-{}", app_id, Uuid::new_v4()); let table = create_local_table(schema, delta_partitions, &topic);