Merge pull request #28 from Sanketika-Obsrv/flink-image-update

#0 Feat: Upgrade flink dependencies and add Flink dockerfiles
Sanketika-Obsrv · Nov 10, 2023 · ffcd442 · ffcd442
2 parents f5528b8 + bf69ff0
commit ffcd442
Show file tree

Hide file tree

Showing 14 changed files with 333 additions and 30 deletions.
diff --git a/.github/workflows/build_and_deploy.yaml b/.github/workflows/build_and_deploy.yaml
@@ -15,7 +15,7 @@ jobs:
         id: tag-checker
         run: |
           (echo -n TRIGGER_ALLOWED= && echo 'print("${{ github.ref_name }}".split("_")[0]
-          in ${{ vars.CURRENT_RELEASE }})' | python3) >> "$GITHUB_OUTPUT"
+          not in ${{ vars.CURRENT_RELEASE }})' | python3) >> "$GITHUB_OUTPUT"
 
   docker-build:
     needs: check-tag

diff --git a/Dockerfile b/Dockerfile
@@ -1,41 +1,41 @@
-FROM --platform=linux/x86_64 maven:3.6.0-jdk-11-slim AS build-core
+FROM --platform=linux/x86_64 maven:3.9.4-eclipse-temurin-11-focal AS build-core
 COPY . /app
 RUN mvn clean install -DskipTests -f /app/framework/pom.xml
 RUN mvn clean install -DskipTests -f /app/dataset-registry/pom.xml
 
-FROM --platform=linux/x86_64 maven:3.6.0-jdk-11-slim AS build-pipeline
+FROM --platform=linux/x86_64 maven:3.9.4-eclipse-temurin-11-focal AS build-pipeline
 COPY --from=build-core /root/.m2 /root/.m2
 COPY . /app
 RUN mvn clean package -DskipTests -f /app/pipeline/pom.xml
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as extractor-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as extractor-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/extractor/target/extractor-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as preprocessor-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as preprocessor-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/preprocessor/target/preprocessor-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as denormalizer-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as denormalizer-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/denormalizer/target/denormalizer-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as transformer-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as transformer-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/transformer/target/transformer-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as router-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as router-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/druid-router/target/druid-router-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-java11 as merged-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as merged-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/pipeline-merged/target/pipeline-merged-1.0.0.jar $FLINK_HOME/lib/
 
-FROM --platform=linux/x86_64 sunbird/flink:1.15.2-scala_2.12-java11 as master-data-processor-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as master-data-processor-image
 USER flink
 COPY --from=build-pipeline /app/pipeline/master-data-processor/target/master-data-processor-1.0.0.jar $FLINK_HOME/lib
 
-FROM --platform=linux/x86_64 sunbird/flink:1.15.2-scala_2.12-java11 as kafka-connector-image
+FROM --platform=linux/x86_64 sanketikahub/flink:1.15.2-scala_2.12-jdk-11 as kafka-connector-image
 USER flink
-COPY --from=build-pipeline /app/pipeline/kafka-connector/target/kafka-connector-1.0.0.jar $FLINK_HOME/lib
+COPY --from=build-pipeline /app/pipeline/kafka-connector/target/kafka-connector-1.0.0.jar $FLINK_HOME/lib
diff --git a/data-products/pom.xml b/data-products/pom.xml
@@ -228,4 +228,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/dataset-registry/pom.xml b/dataset-registry/pom.xml
@@ -36,7 +36,7 @@
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.4</version>
+            <version>2.8.9</version>
         </dependency>
         <dependency>
             <groupId>com.typesafe</groupId>
@@ -178,4 +178,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/framework/pom.xml b/framework/pom.xml
@@ -25,7 +25,6 @@
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
-            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
@@ -51,17 +50,17 @@
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
-            <version>2.4</version>
+            <version>2.8.9</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.12.7</version>
+            <version>2.15.2</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.module</groupId>
             <artifactId>jackson-module-scala_${scala.maj.version}</artifactId>
-            <version>2.12.7</version>
+            <version>2.15.2</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.core</groupId>
@@ -74,11 +73,6 @@
             <artifactId>jedis</artifactId>
             <version>2.9.0</version>
         </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.7.0</version>
-        </dependency>
         <dependency>
             <groupId>com.typesafe</groupId>
             <artifactId>config</artifactId>
@@ -249,4 +243,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/pipeline/druid-router/pom.xml b/pipeline/druid-router/pom.xml
@@ -65,9 +65,17 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
-
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils</artifactId>

diff --git a/pipeline/extractor/pom.xml b/pipeline/extractor/pom.xml
@@ -72,6 +72,17 @@
             <artifactId>embedded-redis</artifactId>
             <version>0.7.1</version>
             <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
@@ -225,4 +236,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/pipeline/kafka-connector/pom.xml b/pipeline/kafka-connector/pom.xml
@@ -50,7 +50,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.datatype</groupId>
             <artifactId>jackson-datatype-joda</artifactId>
-            <version>2.12.7</version>
+            <version>2.15.2</version>
         </dependency>
         <dependency>
             <groupId>org.sunbird.obsrv</groupId>
@@ -229,4 +229,4 @@
         </plugins>
     </build>
 
-</project>
+</project>
diff --git a/pipeline/master-data-processor/pom.xml b/pipeline/master-data-processor/pom.xml
@@ -73,8 +73,17 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
+        </dependency>
         <dependency>
             <groupId>org.json4s</groupId>
             <artifactId>json4s-native_${scala.maj.version}</artifactId>

diff --git a/pipeline/pipeline-merged/pom.xml b/pipeline/pipeline-merged/pom.xml
@@ -83,9 +83,17 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
-
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
             <artifactId>kafka-clients</artifactId>

diff --git a/pipeline/preprocessor/pom.xml b/pipeline/preprocessor/pom.xml
@@ -53,8 +53,17 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+            <version>32.1.2-jre</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>

diff --git a/stubs/docker/apache-flink-plugins/Dockerfile b/stubs/docker/apache-flink-plugins/Dockerfile
@@ -0,0 +1,14 @@
+FROM sanketikahub/flink:1.15.2-scala_2.12-jdk-11-source
+USER flink
+RUN mkdir $FLINK_HOME/plugins/s3-fs-presto
+RUN mkdir $FLINK_HOME/plugins/gs-fs-hadoop
+RUN wget -nv -O flink-streaming-scala_2.12-1.15.2.jar "https://repo1.maven.org/maven2/org/apache/flink/flink-streaming-scala_2.12/1.15.2/flink-streaming-scala_2.12-1.15.2.jar"; \ 
+    mv flink-streaming-scala_2.12-1.15.2.jar $FLINK_HOME/lib/
+# COPY flink-shaded-hadoop2-uber-2.8.3-1.8.3.jar $FLINK_HOME/lib/
+# COPY flink-s3-fs-hadoop-1.15.2.jar $FLINK_HOME/lib/
+RUN wget -nv -O flink-azure-fs-hadoop-1.15.2.jar "https://repo1.maven.org/maven2/org/apache/flink/flink-azure-fs-hadoop/1.15.2/flink-azure-fs-hadoop-1.15.2.jar"; \ 
+    mv flink-azure-fs-hadoop-1.15.2.jar $FLINK_HOME/lib/
+RUN wget -nv -O flink-s3-fs-presto-1.15.2.jar "https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-presto/1.15.2/flink-s3-fs-presto-1.15.2.jar"; \ 
+    mv flink-s3-fs-presto-1.15.2.jar $FLINK_HOME/plugins/s3-fs-presto
+RUN wget -nv -O flink-gs-fs-hadoop-1.15.2.jar "https://repo1.maven.org/maven2/org/apache/flink/flink-gs-fs-hadoop/1.15.2/flink-gs-fs-hadoop-1.15.2.jar"; \ 
+    mv flink-gs-fs-hadoop-1.15.2.jar $FLINK_HOME/plugins/gs-fs-hadoop
diff --git a/stubs/docker/apache-flink/Dockerfile b/stubs/docker/apache-flink/Dockerfile
@@ -0,0 +1,98 @@
+# FLINK SOURCE LINK - https://github.com/apache/flink-docker/blob/4794f9425513fb4c0b55ec1efd629e8eb7e5d8c5/1.15/scala_2.12-java11-ubuntu/Dockerfile
+###############################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM --platform=linux/x86_64 eclipse-temurin:11.0.20.1_1-jdk-focal
+
+RUN apt-get update
+RUN apt-get install libcurl4 curl -y
+
+# Install dependencies
+RUN set -ex; \
+  apt-get update; \
+  apt-get -y install gpg libsnappy1v5 gettext-base libjemalloc-dev; \
+  rm -rf /var/lib/apt/lists/*
+
+# Grab gosu for easy step-down from root
+ENV GOSU_VERSION 1.11
+RUN set -ex; \
+  wget -nv -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)"; \
+  wget -nv -O /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc"; \
+  export GNUPGHOME="$(mktemp -d)"; \
+  for server in ha.pool.sks-keyservers.net $(shuf -e \
+                          hkp://p80.pool.sks-keyservers.net:80 \
+                          keyserver.ubuntu.com \
+                          hkp://keyserver.ubuntu.com:80 \
+                          pgp.mit.edu) ; do \
+      gpg --batch --keyserver "$server" --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4 && break || : ; \
+  done && \
+  gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu; \
+  gpgconf --kill all; \
+  rm -rf "$GNUPGHOME" /usr/local/bin/gosu.asc; \
+  chmod +x /usr/local/bin/gosu; \
+  gosu nobody true
+
+# Configure Flink version
+ENV FLINK_TGZ_URL=https://www.apache.org/dyn/closer.cgi?action=download&filename=flink/flink-1.15.2/flink-1.15.2-bin-scala_2.12.tgz \
+    FLINK_ASC_URL=https://www.apache.org/dist/flink/flink-1.15.2/flink-1.15.2-bin-scala_2.12.tgz.asc \
+    GPG_KEY=0F79F2AFB2351BC29678544591F9C1EC125FD8DB \
+    CHECK_GPG=true
+
+# Prepare environment
+ENV FLINK_HOME=/opt/flink
+ENV PATH=$FLINK_HOME/bin:$PATH
+RUN groupadd --system --gid=9999 flink && \
+    useradd --system --home-dir $FLINK_HOME --uid=9999 --gid=flink flink
+WORKDIR $FLINK_HOME
+
+# Install Flink
+RUN set -ex; \
+  wget -nv -O flink.tgz "$FLINK_TGZ_URL"; \
+  \
+  if [ "$CHECK_GPG" = "true" ]; then \
+    wget -nv -O flink.tgz.asc "$FLINK_ASC_URL"; \
+    export GNUPGHOME="$(mktemp -d)"; \
+    for server in ha.pool.sks-keyservers.net $(shuf -e \
+                            hkp://p80.pool.sks-keyservers.net:80 \
+                            keyserver.ubuntu.com \
+                            hkp://keyserver.ubuntu.com:80 \
+                            pgp.mit.edu) ; do \
+        gpg --batch --keyserver "$server" --recv-keys "$GPG_KEY" && break || : ; \
+    done && \
+    gpg --batch --verify flink.tgz.asc flink.tgz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" flink.tgz.asc; \
+  fi; \
+  \
+  tar -xf flink.tgz --strip-components=1; \
+  rm flink.tgz; \
+  \
+  chown -R flink:flink .; \
+  \
+  # Replace default REST/RPC endpoint bind address to use the container's network interface \
+  sed -i 's/rest.address: localhost/rest.address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/rest.bind-address: localhost/rest.bind-address: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/jobmanager.bind-host: localhost/jobmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i 's/taskmanager.bind-host: localhost/taskmanager.bind-host: 0.0.0.0/g' $FLINK_HOME/conf/flink-conf.yaml; \
+  sed -i '/taskmanager.host: localhost/d' $FLINK_HOME/conf/flink-conf.yaml;
+
+# Configure container
+COPY docker-entrypoint.sh /
+ENTRYPOINT ["/docker-entrypoint.sh"]
+EXPOSE 6123 8081
+CMD ["help"]