Merge branch 'temp/IT_Load_Test' of https://github.com/ollionorg/Data…

…flowTemplates-fork into temp/IT_Load_Test

.github/actions/setup-env/action.yml

@@ -35,7 +35,7 @@ inputs:
     type: string
     description: 'The version of Java to install'
     required: false
-    default: '11'
+    default: '17'
   go-version:
     type: string
     description: 'The version of Go to install'

.github/actions/setup-java-env/action.yml

@@ -30,7 +30,7 @@ inputs:
     type: string
     description: 'The version of Java to install'
     required: false
-    default: '11'
+    default: '17'
 outputs:
   cache-hit:
     description: 'Whether or not there was a cache hit'

.github/workflows/spanner-staging-tests.yml

@@ -15,6 +15,17 @@
 name: Spanner Staging integration tests
 
 on:
+  pull_request:
+    branches:
+    - 'main'
+    paths:
+    - '.github/workflows/spanner-staging-tests.yml'
+    inputs:
+      spannerHost:
+        description: 'Spanner host URL.'
+        type: string
+        required: false
+        default: 'https://staging-wrenchworks.sandbox.googleapis.com'
   workflow_dispatch:
     inputs:
       commitOrTag:
@@ -35,7 +46,7 @@ jobs:
     name: Spanner Dataflow Templates Integration Tests
     timeout-minutes: 180
     # Run on any runner that matches all the specified runs-on values.
-    runs-on: [ self-hosted, it ]
+    runs-on: [ self-hosted, spanner ]
     steps:
     - name: Checkout Code
       uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 # v4.0.0
@@ -47,14 +58,13 @@ jobs:
     - name: Run Integration Tests
       run: |
         ./cicd/run-spanner-staging-it-tests \
-        --modules-to-build="ALL" \
+        --modules-to-build="v1" \
         --it-region="us-central1" \
-        --it-project="cloud-teleport-testing" \
-        --it-artifact-bucket="cloud-teleport-testing-it-gitactions" \
-        --it-private-connectivity="datastream-private-connect-us-central1" \
+        --it-project="span-cloud-migrations-staging" \
+        --it-artifact-bucket="dataflow-testing-it-spanner-staging" \
         --it-spanner-host=${{ inputs.spannerHost }}
     - name: Upload Integration Tests Report
-      uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+      uses: actions/upload-artifact@v4
       if: always() # always run even if the previous step fails
       with:
         name: surefire-test-results

contributor-docs/code-contributions.md

@@ -28,7 +28,7 @@ these can be found in the following locations:
 
 ### Requirements
 
-* Java 11
+* Java 17
 * Maven 3
 * IntelliJ (recommended) or another editor of your choice
 

it/google-cloud-platform/src/main/java/org/apache/beam/it/gcp/artifacts/matchers/ArtifactsSubject.java

@@ -36,6 +36,8 @@
 import org.apache.beam.it.gcp.artifacts.utils.JsonTestUtil;
 import org.apache.beam.it.gcp.artifacts.utils.ParquetTestUtil;
 import org.apache.beam.it.truthmatchers.RecordsSubject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
  * Subject that has assertion operations for artifact lists (GCS files), usually coming from the
@@ -46,6 +48,8 @@
 })
 public final class ArtifactsSubject extends Subject {
 
+  private static final Logger LOG = LoggerFactory.getLogger(ArtifactsSubject.class);
+
   private final List<Artifact> actual;
 
   private static final ObjectMapper objectMapper = new ObjectMapper();
@@ -96,8 +100,15 @@ public void hasContent(String content) {
    * @param hash Content to search for
    */
   public void hasHash(String hash) {
+
     if (actual.stream()
         .noneMatch(artifact -> sha256().hashBytes(artifact.contents()).toString().equals(hash))) {
+      actual.stream()
+          .forEach(
+              artifact -> {
+                String calculatedHash = sha256().hashBytes(artifact.contents()).toString();
+                System.out.printf("Calculated Hash (no match found): {%s} \n", calculatedHash);
+              });
       failWithActual("expected to contain hash", hash);
     }
   }

metadata/pom.xml

@@ -34,8 +34,8 @@
   <url>https://github.com/GoogleCloudPlatform/DataflowTemplates</url>
 
   <properties>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
   </properties>
 
   <dependencies>

plaintext-logging/pom.xml

@@ -27,7 +27,7 @@
   <artifactId>plaintext-logging</artifactId>
 
   <properties>
-    <java.version>11</java.version>
+    <java.version>17</java.version>
     <surefire.version>2.21.0</surefire.version>
   </properties>
 

plugins/core-plugin/pom.xml

@@ -32,8 +32,8 @@
   <url>https://github.com/GoogleCloudPlatform/DataflowTemplates</url>
 
   <properties>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
 
     <reflections.version>0.9.11</reflections.version>
     <gson.version>2.9.1</gson.version>

plugins/core-plugin/src/main/java/com/google/cloud/teleport/plugin/DockerfileGenerator.java

@@ -43,7 +43,7 @@
 public class DockerfileGenerator {
 
   public static final String BASE_CONTAINER_IMAGE =
-      "gcr.io/dataflow-templates-base/java11-template-launcher-base-distroless:latest";
+      "gcr.io/dataflow-templates-base/java17-template-launcher-base-distroless:latest";
   // Keep in sync with python version used in
   // https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/python/generate_dependencies.sh
   public static final String BASE_PYTHON_CONTAINER_IMAGE =

plugins/templates-maven-plugin/pom.xml

@@ -36,12 +36,12 @@
   </prerequisites>
 
   <properties>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
-    <maven.version>3.3.9</maven.version>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
+    <maven.version>3.6.3</maven.version>
     <dataflow.api.version>0.7.6</dataflow.api.version>
     <mojo.executor.version>2.4.0</mojo.executor.version>
-    <maven-plugin.version>3.6.0</maven-plugin.version>
+    <maven-plugin.version>3.15.1</maven-plugin.version>
   </properties>
 
   <dependencies>

plugins/templates-maven-plugin/src/main/java/com/google/cloud/teleport/plugin/maven/TemplatesReleaseFinishMojo.java

@@ -85,7 +85,7 @@ public class TemplatesReleaseFinishMojo extends TemplatesBaseMojo {
   @Parameter(
       name = "baseContainerImage",
       defaultValue =
-          "gcr.io/dataflow-templates-base/java11-template-launcher-base-distroless:latest",
+          "gcr.io/dataflow-templates-base/java17-template-launcher-base-distroless:latest",
       required = false)
   protected String baseContainerImage;
 

pom.xml

@@ -31,13 +31,13 @@
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <java.version>11</java.version>
+    <java.version>17</java.version>
 <!--    <os.detected.classifier>osx-x86_64</os.detected.classifier>-->
     <!-- Plugins -->
     <templates-maven-plugin.version>1.0-SNAPSHOT</templates-maven-plugin.version>
     <maven-checkstyle-plugin.version>3.2.1</maven-checkstyle-plugin.version>
     <maven-compiler-plugin.version>3.11.0</maven-compiler-plugin.version>
-    <maven-dependency-plugin.version>3.6.0</maven-dependency-plugin.version>
+    <maven-dependency-plugin.version>3.8.1</maven-dependency-plugin.version>
     <maven-enforcer-plugin.version>3.5.0</maven-enforcer-plugin.version>
     <extra.enforcer.rules.version>1.8.0</extra.enforcer.rules.version>
     <maven-jar-plugin.version>3.3.0</maven-jar-plugin.version>
@@ -48,8 +48,8 @@
     <jacoco.version>0.8.8</jacoco.version>
 
     <!-- Beam and linked versions -->
-    <beam.version>2.61.0</beam.version>
-    <beam-python.version>2.61.0</beam-python.version>
+    <beam.version>2.62.0</beam.version>
+    <beam-python.version>2.62.0</beam-python.version>
     <beam-maven-repo></beam-maven-repo>
 
     <!-- Common dependency versions -->
@@ -66,7 +66,7 @@
     <failsafe.version>3.3.0</failsafe.version>
     <grpc.gen.version>1.13.1</grpc.gen.version>
     <guava.version>32.0.1-jre</guava.version>
-    <hadoop.version>3.4.0</hadoop.version>
+    <hadoop.version>3.4.1</hadoop.version>
     <hamcrest.version>2.1</hamcrest.version>
     <hbase.client.version>2.5.3-hadoop3</hbase.client.version>
     <mockito.version>4.11.0</mockito.version>
@@ -77,14 +77,14 @@
     <junit.version>4.13.2</junit.version>
     <re2j.version>1.6</re2j.version>
     <slf4j.version>1.7.36</slf4j.version>
-    <snakeyaml.version>2.0</snakeyaml.version>
+    <snakeyaml.version>2.2</snakeyaml.version>
     <snappy.version>1.1.10.4</snappy.version>
     <spotless-maven-plugin.version>2.40.0</spotless-maven-plugin.version>
     <surefire.version>2.21.0</surefire.version>
     <truth.version>1.1.5</truth.version>
 
     <!-- TODO(https://github.com/apache/beam/issues/30700): Remove once Beam gets the latest version of gax. -->
-    <gax.version>2.48.0</gax.version>
+    <gax.version>2.55.0</gax.version>
 
     <!-- Drop pinned version once maven-dependency-plugin gets past plexus-archiver 4.8.0 -->
     <plexus-archiver.version>4.8.0</plexus-archiver.version>

python/pom.xml

@@ -34,8 +34,8 @@
   <url>https://github.com/GoogleCloudPlatform/DataflowTemplates</url>
 
   <properties>
-    <maven.compiler.source>11</maven.compiler.source>
-    <maven.compiler.target>11</maven.compiler.target>
+    <maven.compiler.source>17</maven.compiler.source>
+    <maven.compiler.target>17</maven.compiler.target>
 
     <!-- TODO - remove line when repo is upgraded to Beam 2.59.0 -->
     <beam-python.version>2.57.0</beam-python.version>