Prepare CI for testing.

Fix pass secret env to docker build. Fix maven auth. Rework docker compose file. Add mout directories with jars. Add copying jars to /app directories to further mounting. Update .gitignore. Fix extender's tests. Fix remote builder mappings.
defold · Jun 20, 2024 · eaa6033 · eaa6033
1 parent fdad63d
commit eaa6033
Show file tree

Hide file tree

Showing 19 changed files with 125 additions and 148 deletions.
diff --git a/.github/workflows/application-build.yml b/.github/workflows/application-build.yml
@@ -1,12 +1,17 @@
 name: Build jars
 
 on:
+  workflow_dispatch:
   push:
     tags:
       - "extender-v[0-9]+.[0-9]+.[0-9]+"
       - "client-v[0-9]+.[0-9]+.[0-9]+"
       - "manifestmerger-v[0-9]+.[0-9]+.[0-9]+"
 
+env:
+  GOOGLE_ARTIFACT_REGISTRY: europe-north1-docker.pkg.dev
+  MAVEN_SERVICE_ACCOUNT: superb-chamois-registry@extender-poc.iam.gserviceaccount.com
+
 jobs:
   client:
     if: ${{ startsWith(github.ref_name, 'client-') }}
@@ -22,6 +27,13 @@ jobs:
           TAG_VERSION="${TAG_NAME#*-v}"
 
           echo "tag_version=$TAG_VERSION" >> $GITHUB_OUTPUT
+    - name: Login to Google Artifact registry (Maven)
+      uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa
+      with:
+        service_account: ${{ env.MAVEN_SERVICE_ACCOUNT }}
+        credentials_json: ${{ secrets.MAVEN_REGISTRY_JSON_KEY }}
+        export_environment_variables: true
+        create_credentials_file: true
     - name: Install Java
       uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
       with:
@@ -30,7 +42,7 @@ jobs:
     - name: Setup Gradle
       uses: gradle/actions/setup-gradle@31ae3562f68c96d481c31bc1a8a55cc1be162f83
     - name: Build the extender-client.jar
-      run: ./gradlew -PclientVersion=${{ steps.version.outputs.tag_version }} client:build
+      run: ./gradlew -PclientVersion=${{ steps.version.outputs.tag_version }} client:jar
     - name: Test the extender-client.jar
       run: ./gradlew -PclientVersion=${{ steps.version.outputs.tag_version }} client:test
     - name: Publish to Google artifact registry
@@ -42,15 +54,15 @@ jobs:
           ${{ github.workspace }}/gh_release/extender-client-${{ steps.version.outputs.tag_version }}.jar
     - name: Publish to Github release
       run: |
-        chmod 0444 ${{ github.workspace }}/*.jar
+        chmod 0444 ${{ github.workspace }}/gh_release/*.jar
           echo ${{ github.token }} | gh auth login --with-token
           gh release create "${GITHUB_REF_NAME}"  \
             --draft \
             --generate-notes \
             --target ${{ github.sha }} \
             ${{ github.workspace }}/gh_release/*.jar
     - name: Notify if tests failed
-      uses: homoluctus/slatify@master
+      uses: homoluctus/slatify@cd4b4a1158cfb3e26fe1ee35c1cd4f0247dfbf96
       if: failure()
       with:
         type: ${{ job.status }}
@@ -72,6 +84,13 @@ jobs:
           TAG_VERSION="${TAG_NAME#*-v}"
 
           echo "tag_version=$TAG_VERSION" >> $GITHUB_OUTPUT
+    - name: Login to Google Artifact registry (Maven)
+      uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa
+      with:
+        service_account: ${{ env.MAVEN_SERVICE_ACCOUNT }}
+        credentials_json: ${{ secrets.MAVEN_REGISTRY_JSON_KEY }}
+        export_environment_variables: true
+        create_credentials_file: true
     - name: Install Java
       uses: actions/setup-java@99b8673ff64fbf99d8d325f52d9a5bdedb8483e9
       with:
@@ -92,15 +111,15 @@ jobs:
           ${{ github.workspace }}/gh_release/manifestmergetool-${{ steps.version.outputs.tag_version }}.jar
     - name: Publish to Github release
       run: |
-        chmod 0444 ${{ github.workspace }}/*.jar
+        chmod 0444 ${{ github.workspace }}/gh_release/*.jar
           echo ${{ github.token }} | gh auth login --with-token
           gh release create "${GITHUB_REF_NAME}"  \
             --draft \
             --generate-notes \
             --target ${{ github.sha }} \
             ${{ github.workspace }}/gh_release/*.jar
     - name: Notify if tests failed
-      uses: homoluctus/slatify@master
+      uses: homoluctus/slatify@cd4b4a1158cfb3e26fe1ee35c1cd4f0247dfbf96
       if: failure()
       with:
         type: ${{ job.status }}
@@ -110,8 +129,6 @@ jobs:
 
   extender:
     if: ${{ startsWith(github.ref_name, 'extender-') }}
-    env:
-      GOOGLE_ARTIFACT_REGISTRY: europe-north1-docker.pkg.dev
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repo
@@ -147,11 +164,12 @@ jobs:
     - name: Run extender tests
       run: ./gradlew -PextenderVersion=${{ steps.version.outputs.tag_version }} server:test -i
     - name: Login to Google Artifact registry (Maven)
-      uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
+      uses: google-github-actions/auth@71fee32a0bb7e97b4d33d548e7d957010649d8fa
       with:
-        registry: ${{ env.GOOGLE_ARTIFACT_REGISTRY }}
-        username: _json_key
-        password: ${{ secrets.MAVEN_REGISTRY_JSON_KEY }}
+        service_account: ${{ env.MAVEN_SERVICE_ACCOUNT }}
+        credentials_json: ${{ secrets.MAVEN_REGISTRY_JSON_KEY }}
+        export_environment_variables: true
+        create_credentials_file: true
     - name: Publish to Google artifact registry
       run: ./gradlew -PextenderVersion=${{ steps.version.outputs.tag_version }} publishExtenderPublicationToExtenderRepoRepository
     - name: Prepare jar for GH release
@@ -161,15 +179,15 @@ jobs:
           ${{ github.workspace }}/gh_release/extender-${{ steps.version.outputs.tag_version }}.jar
     - name: Publish to Github release
       run: |
-        chmod 0444 ${{ github.workspace }}/*.jar
+        chmod 0444 ${{ github.workspace }}/gh_release/*.jar
           echo ${{ github.token }} | gh auth login --with-token
           gh release create "${GITHUB_REF_NAME}"  \
             --draft \
             --generate-notes \
             --target ${{ github.sha }} \
             ${{ github.workspace }}/gh_release/*.jar
     - name: Notify if tests failed
-      uses: homoluctus/slatify@master
+      uses: homoluctus/slatify@cd4b4a1158cfb3e26fe1ee35c1cd4f0247dfbf96
       if: failure()
       with:
         type: ${{ job.status }}

diff --git a/.github/workflows/docker-env-build.yml b/.github/workflows/docker-env-build.yml
@@ -1,9 +1,20 @@
 name: Build Docker environment container
 
 on:
+  workflow_dispatch:
   push:
     tags:
-      - "*"
+      - "android*"
+      - "base*"
+      - "linux*"
+      - "emsdk*"
+      - "wine*"
+      - "winsdk*"
+      - "nssdk*"
+      - "ps4*"
+      - "ps5*"
+
+
 
 env:
   GOOGLE_ARTIFACT_REGISTRY: europe-north1-docker.pkg.dev
@@ -19,9 +30,9 @@ jobs:
     - name: Process tag name
       id: name_postprocess
       run: |
-          PRIVATE_PLATFORM_LIST=${{ env.PRIVATE_PLATFORMS }}
-          PUBLIC_REGISTRY=${{ env.PUBLIC_REGISTRY }}
-          PRIVATE_REGISTRY=${{ env.PRIVATE_REGISTRY }}
+          PRIVATE_PLATFORM_LIST="${{ env.PRIVATE_PLATFORMS }}"
+          PUBLIC_REGISTRY="${{ env.PUBLIC_REGISTRY }}"
+          PRIVATE_REGISTRY="${{ env.PRIVATE_REGISTRY }}"
 
           SHORT_SHA="$(echo ${GITHUB_SHA} | cut -c1-8)"
           TAG_NAME=${{ github.ref_name }}
@@ -66,7 +77,7 @@ jobs:
         platforms: |
           linux/amd64
         secret-envs: |
-          id=DM_PACKAGES_URL
+          DM_PACKAGES_URL=DM_PACKAGES_URL
         tags: |
           ${{ steps.name_postprocess.outputs.target_registry }}/extender-${{ steps.name_postprocess.outputs.image_name }}-env:${{ steps.name_postprocess.outputs.image_version }}
           ${{ steps.name_postprocess.outputs.target_registry }}/extender-${{ steps.name_postprocess.outputs.image_name }}-env:${{ steps.name_postprocess.outputs.short_sha }}

diff --git a/.github/workflows/dockerimage.yml b/.github/workflows/dockerimage.yml
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,9 @@ server/test-data/sdk/*
 .bash_history
 
 Version.java
-/server/docker/*.jar
+/server/app/*
+server/bin/*
+server/manifestmergetool/app/*
+server/manifestmergetool/bin/*
+
+.vscode/*
diff --git a/client/build.gradle b/client/build.gradle
@@ -17,7 +17,6 @@ publishing {
       groupId 'com.defold.extender'
       artifactId 'client'
       version project.ext.clientVersion
-      from components.java
     }
   }
 }

diff --git a/server/build-docker.sh b/server/build-docker.sh
@@ -3,16 +3,6 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
-# $SCRIPT_DIR/../gradlew build -xtest
-# $SCRIPT_DIR/../gradlew :manifestmergetool:mainJar
-
-# # copy jars
-# cp $SCRIPT_DIR/build/libs/extender-0.1.0.jar $SCRIPT_DIR/docker
-# cp $SCRIPT_DIR/manifestmergetool/build/libs/manifestmergetool-0.1.0.jar $SCRIPT_DIR/docker
-
-# # copy users
-# cp -r $SCRIPT_DIR/users $SCRIPT_DIR/docker
-
 DOCKER_REGISTRY=europe-north1-docker.pkg.dev/extender-poc/extender-public-registry
 DOCKER_PRIVATE_REGISTRY=europe-north1-docker.pkg.dev/extender-poc/extender-private-registry
 

diff --git a/server/build.gradle b/server/build.gradle
@@ -118,6 +118,15 @@ compileJava {
 bootJar {
     archiveBaseName = 'extender'
     version = project.ext.extenderVersion
+    doLast {
+        copy {
+            from "${project.projectDir}/build/libs/extender-${project.ext.extenderVersion}.jar"
+            into "app"
+            rename { String fileName ->
+                fileName.replace("-$project.ext.extenderVersion", "")
+            }
+        }
+    }
 }
 
 // Builds the extender server base classes into a .jar file
@@ -134,7 +143,6 @@ publishing {
       groupId 'com.defold.extender'
       artifactId 'server'
       version project.ext.extenderVersion
-      from components.java
     }
   }
 }

diff --git a/server/configs/application-local-dev-app.yml b/server/configs/application-local-dev-app.yml
@@ -2,27 +2,20 @@ extender:
     remote-builder:
         enabled: true
         platforms:
-            android-ndk25: http://android_ndk25:9000
-            android-latest: http://android_ndk25:9000
-            emsdk-2011: http://emscripten_2011:9000
-            emsdk-3155: http://emscripten_3155:9000
-            emsdk-latest: http://emscripten_3155:9000
+            android-ndk25: http://android-ndk25:9000
+            android-latest: http://android-ndk25:9000
+            emsdk-2011: http://emsdk-2011:9000
+            emsdk-3155: http://emsdk-3155:9000
+            emsdk-latest: http://emsdk-3155:9000
             linux-latest: http://linux:9000
-            nssdk-1532: http://nssdk_1532:9000
-            nssdk-1753: http://nssdk_1753:9000
-            nssdk-latest: http://nssdk_1753:9000
-            ps4-10500: http://ps4_10500:9000
-            ps4-11000: http://ps4_11000:9000
-            ps4-latest: http://ps4_11000:9000
-            ps5-8000: http://ps5_8000:9000
-            ps5-latest: http://ps5_8000:9000
-            winsdk-2019: http://win_2019:9000
-            winsdk-2022: http://win_2022:9000
-            winsdk-latest: http://win_2022:9000
-
-spring:
-    servlet:
-        multipart:
-            enabled: true
-            max-file-size: 512MB
-            max-request-size: 512MB
+            nssdk-1532: http://nssdk-1532:9000
+            nssdk-1753: http://nssdk-1753:9000
+            nssdk-latest: http://nssdk-1753:9000
+            ps4-10500: http://ps4-10500:9000
+            ps4-11000: http://ps4-11000:9000
+            ps4-latest: http://ps4-11000:9000
+            ps5-8000: http://ps5-8000:9000
+            ps5-latest: http://ps5-8000:9000
+            winsdk-2019: http://winsdk-2019:9000
+            winsdk-2022: http://winsdk-2022:9000
+            winsdk-latest: http://winsdk-2022:9000
diff --git a/server/docker/Dockerfile.base-app b/server/docker/Dockerfile.base-app
diff --git a/server/docker/common-services.yml b/server/docker/common-services.yml
@@ -2,14 +2,19 @@ services:
   test_builder:
     platform: linux/amd64
     volumes:
+      - ./../app/:/app/:ro
       - ./../configs:/etc/defold/extender:ro
       - ./../test-data/sdk:/var/extender/sdk
+    entrypoint: ["java","-Xmx4g","-XX:MaxDirectMemorySize=2g","-jar","/app/extender.jar"]
     environment:
       - EXAMPLE_VAR=1
   common_builder:
     platform: linux/amd64
     volumes:
+      - ./../app/:/app/:ro
+      - ./../manifestmergetool/app/:/opt/local/bin/:ro
       - ./../configs:/etc/defold/extender:ro
+    entrypoint: ["java","-Xmx4g","-XX:MaxDirectMemorySize=2g","-jar","/app/extender.jar"]
     environment:
       - EXAMPLE_VAR=1
       # - DM_DEBUG_COMMANDS=1