From e0ec0063c15cac3e980839d2343c6468f3758dc8 Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Fri, 15 Mar 2024 10:09:00 -0700 Subject: [PATCH 1/3] chore: update the Dockerfiles to build more cleanly Now with `Makefile`: `make docker` Fixes #170 --- .dockerignore | 4 +++- Dockerfile | 13 +++++------ Dockerfile.Debian | 17 -------------- Makefile | 22 +++++++++++++++++++ Dockerfile.setup => contrib/Dockerfile.setup | 0 .../docker-compose-with-statsd.yml | 0 docker-compose.yml | 2 +- src/dead_letters.rs | 2 +- 8 files changed, 33 insertions(+), 27 deletions(-) delete mode 100644 Dockerfile.Debian create mode 100644 Makefile rename Dockerfile.setup => contrib/Dockerfile.setup (100%) rename docker-compose-with-statsd.yml => contrib/docker-compose-with-statsd.yml (100%) diff --git a/.dockerignore b/.dockerignore index b12c6f8..dd59f6c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,5 @@ .git/ +contrib/ target/ -.github/ \ No newline at end of file +venv/ +.github/ diff --git a/Dockerfile b/Dockerfile index 85b9094..9de6e98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,16 @@ -FROM rust:1.67 AS builder +FROM rust:1.75 AS builder +RUN mkdir /build WORKDIR /build - COPY ./ . -RUN cargo build --release +RUN cargo build --release --features s3 -FROM ubuntu +FROM alpine -RUN apt-get update -RUN apt-get install -y ca-certificates +RUN apk add -U ca-certificates WORKDIR /build COPY --from=builder /build/target/release/kafka-delta-ingest ./ -ENTRYPOINT ["/build/kafka-delta-ingest"] \ No newline at end of file +ENTRYPOINT ["/build/kafka-delta-ingest"] diff --git a/Dockerfile.Debian b/Dockerfile.Debian deleted file mode 100644 index 919ea7b..0000000 --- a/Dockerfile.Debian +++ /dev/null @@ -1,17 +0,0 @@ -FROM rust:1.67.0-bullseye as builder -WORKDIR /usr/src/kafka-delta-ingest -COPY . . -RUN cargo install --path . - -FROM debian:bullseye -RUN apt-get update && \ - apt-get install -qy pkg-config libssl-dev ca-certificates && \ - apt-get autoremove -y && \ - rm -rf /var/apt/lists/* && \ - rm -rf /var/cache/apt/* && \ - rm -rf /var/lib/apt/lists/* -COPY --from=builder /usr/local/cargo/bin/kafka-delta-ingest /usr/local/bin/kafka-delta-ingest - -RUN kafka-delta-ingest -V - -ENTRYPOINT ["kafka-delta-ingest"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a1ac58c --- /dev/null +++ b/Makefile @@ -0,0 +1,22 @@ +.DEFAULT_GOAL := help + +CARGO:=$(shell which cargo) +APP=target/debug/placementd-web +SOURCES=$(shell find . -type f -iname '*.rs') + +$(APP): Cargo.toml $(SOURCES) + $(CARGO) build + +.PHONY: check +check: Cargo.toml $(SOURCES) + $(CARGO) fmt + $(CARGO) test --features azure,s3 + +.PHONY: docker +docker: Dockerfile ## Build the docker image + docker build -t kafka-delta-ingest . + + +.PHONY: help +help: + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/Dockerfile.setup b/contrib/Dockerfile.setup similarity index 100% rename from Dockerfile.setup rename to contrib/Dockerfile.setup diff --git a/docker-compose-with-statsd.yml b/contrib/docker-compose-with-statsd.yml similarity index 100% rename from docker-compose-with-statsd.yml rename to contrib/docker-compose-with-statsd.yml diff --git a/docker-compose.yml b/docker-compose.yml index fa6d878..796d384 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: setup: build: context: . - dockerfile: Dockerfile.setup + dockerfile: contrib/Dockerfile.setup depends_on: - kafka - localstack diff --git a/src/dead_letters.rs b/src/dead_letters.rs index a4e709e..d1acec7 100644 --- a/src/dead_letters.rs +++ b/src/dead_letters.rs @@ -254,7 +254,7 @@ impl DeltaSinkDeadLetterQueue { dynamo_lock_options::DYNAMO_LOCK_PARTITION_KEY_VALUE.to_string() => std::env::var(env_vars::DEAD_LETTER_DYNAMO_LOCK_PARTITION_KEY_VALUE) .unwrap_or_else(|_| "kafka_delta_ingest-dead_letters".to_string()), }; - #[cfg(feature = "azure")] + #[cfg(all(feature = "azure", not(feature = "s3")))] let opts = HashMap::default(); let table = crate::delta_helpers::load_table(table_uri, opts.clone()).await?; From f36f2b7ce24dc8d0cefa790e2c064f7b0de4dcaa Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Sat, 25 May 2024 05:23:13 +0000 Subject: [PATCH 2/3] chore: pin to an older Python image which has aws-cli available --- contrib/Dockerfile.setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Dockerfile.setup b/contrib/Dockerfile.setup index f369163..ae7b484 100644 --- a/contrib/Dockerfile.setup +++ b/contrib/Dockerfile.setup @@ -1,4 +1,4 @@ -FROM python:alpine +FROM python:3-alpine3.18 RUN apk add --no-cache aws-cli bash openssh ca-certificates jq curl openssl perl git zip \ && apk add --no-cache --virtual .build-deps gcc make openssl-dev libffi-dev musl-dev linux-headers \ From 1841f7832a59aef958c0727203568d0e1ea560cd Mon Sep 17 00:00:00 2001 From: "R. Tyler Croy" Date: Sat, 25 May 2024 05:35:31 +0000 Subject: [PATCH 3/3] remove unused code --- src/coercions.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/coercions.rs b/src/coercions.rs index c69d255..4b2ec42 100644 --- a/src/coercions.rs +++ b/src/coercions.rs @@ -26,11 +26,6 @@ pub(crate) struct CoercionTree { root: HashMap, } -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct CoercionArray { - element: CoercionNode, -} - /// Returns a [`CoercionTree`] so the schema can be walked efficiently level by level when performing conversions. pub(crate) fn create_coercion_tree(schema: &DeltaSchema) -> CoercionTree { let mut root = HashMap::new();