From a52f241f2a7e2ca2a9f706b607258778ba4da37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn-Andre=20Skaar?= <31540110+bjornandre@users.noreply.github.com> Date: Tue, 14 Nov 2023 12:23:57 +0100 Subject: [PATCH] Fixed repo config for Maven and add docker build (#62) * Configure cache directly in actions/setup-java * Remove build config for Azure Pipelines and Cloud Build * Fixed documentation since targetUri option and export endpoint is removed * Set up docker build --- .github/workflows/build.yml | 66 ++++- Dockerfile | 2 +- README.md | 46 +-- azure-pipelines.yml | 47 --- cloudbuild.yml | 21 -- pom.xml | 271 ++++++++++-------- .../service/pseudo/PseudoController.java | 8 +- 7 files changed, 209 insertions(+), 252 deletions(-) delete mode 100644 azure-pipelines.yml delete mode 100644 cloudbuild.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 411675f..a757609 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,9 +4,22 @@ on: push: branches: - master + paths: + - src/** + - conf/** + - Dockerfile pull_request: branches: - master + paths: + - src/** + - conf/** + - Dockerfile + +env: + REGISTRY: europe-north1-docker.pkg.dev/artifact-registry-5n/dapla-pseudo-docker/ssb/dapla + IMAGE: dapla-dlp-pseudo-service + TAG: ${{ github.ref_name }}-${{ github.sha }} jobs: build: @@ -23,6 +36,7 @@ jobs: with: java-version: 21 distribution: zulu + cache: maven - name: Authenticate to Google Cloud id: auth @@ -32,13 +46,51 @@ jobs: service_account: "gh-actions-dapla-pseudo@artifact-registry-5n.iam.gserviceaccount.com" token_format: access_token - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ runner.os }}-m2 - - name: Build with Maven and deploy to Artifact Registry run: mvn --batch-mode -P ssb-bip deploy + - name: Clean up artifacts that are no longer needed + run: | + rm -f target/dapla-dlp-pseudo-service-*-sources.jar + rm -f target/dapla-dlp-pseudo-service-*-javadoc.jar + ls -al target/dapla-dlp-pseudo-service-*.jar + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Artifact Registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: "oauth2accesstoken" + password: "${{ steps.auth.outputs.access_token }}" + + - name: Docker meta + id: metadata + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE }} + # Docker tags based on the following events/attributes + tags: | + type=ref,event=branch + type=ref,event=pr + type=raw,value=latest,enable={{is_default_branch}} + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}} + type=semver,pattern=v{{major}} + type=raw,value=${{ env.TAG }}, enable=true + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v4 + with: + file: Dockerfile + push: true + context: . + tags: | + ${{ steps.metadata.outputs.tags }} + labels: ${{ steps.metadata.outputs.labels }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b7f125c..76b280f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM azul/zulu-openjdk:17 +FROM azul/zulu-openjdk:21 RUN apt-get -qq update && apt-get -y dist-upgrade && apt-get -y --no-install-recommends install curl COPY target/dapla-dlp-pseudo-service-*.jar dapla-dlp-pseudo-service.jar COPY target/classes/logback*.xml /conf/ diff --git a/README.md b/README.md index 160dea3..f8f1985 100644 --- a/README.md +++ b/README.md @@ -11,49 +11,6 @@ Browse the API docs as: ## Examples -### Export a dataset -```sh -curl "${root_url}/export" -i -H "Authorization: Bearer ${dapla_auth_token}" --data @export-request.json -``` -Where `root_url` points to an instance of the pseudo-service, `dapla_auth_token` is a JWT token and -`export-request.json` is a file containing the request. E.g: - -```json -{ - "sourceDataset": { - "root": "gs://ssb-dev-demo-enhjoern-a-data-produkt", - "path": "/path/to/data", - "version": "123" - }, - "targetContentName": "test", - "targetContentType": "application/json", - "targetPassword": "kensentme", - "depseudonymize": true, - "pseudoRules": [ - { - "name": "kontonummer", - "pattern": "**/kontonummer", - "func": "fpe-anychar(secret1)" - } - ] -} -``` - -This example exports all columns matching either `**/foedsel` or `**/kontonummer` from a dataset located in a GCS -bucket at `gs://ssb-dev-demo-enhjoern-a-data-produkt/path/to/data/123`. -Columns matching `**/kontonummer` will be depseudonymized using the function `fpe-anychar(secret1)` and then compressed, -encrypted and uploaded (as json) to the preconfigured data export bucket (see config). - -Note that the above will export all data. If you only need a subset of fields, you can specify this with column selector -glob expressions, like so: -``` - "columnSelectors": [ - "**/foedsel*", - "**/kontonummer" - ] -``` - - ### Pseudonymize JSON file and stream back the result ```sh @@ -94,13 +51,12 @@ curl "${root_url}/depseudonymize/file" \ }' ``` -### Depseudonymize JSON file and upload to google cloud storage as zipped CSV-file +### Depseudonymize JSON file and download a zipped CSV-file ```sh curl "${root_url}/depseudonymize/file" \ --header "Authorization: Bearer ${dapla_auth_token}" \ --form 'data=@src/test/resources/data/15k-pseudonymized.json' \ --form 'request={ - "targetUri": "gs://ssb-dev-demo-enhjoern-a-data-export/path/to/depseudonymized-csv.zip", "targetContentType": "text/csv", "pseudoConfig": { "rules": [ diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 12553b3..0000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: $(Build.SourceBranch)-$(date:yyyyMMdd)$(rev:.r) -# Pipeline triggers on any branch and tag -trigger: - branches: - include: - - '*' - tags: - include: - - '*' - -resources: - repositories: - - repository: templates - type: github - name: statisticsnorway/azure-pipelines-templates - ref: refs/heads/java-app-complete-build - endpoint: statisticsnorway - -# Pipeline will be run on this base image -pool: - vmImage: 'ubuntu-latest' - -# Variables global to this pipeline -# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/variables -variables: - # Variables defined in Pipelines->Library->Variable Groups in your project in - # Azure Pipelines - - group: Hemmeligheter - # Variables defined here - - name: MAVEN_CACHE_FOLDER - value: $(Pipeline.Workspace)/.m2/repository - - name: MAVEN_OPTS - value: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER)' - - -# Job which will run mvn build, mvn install and Sonar Analysis (if branch is master) -jobs: - - - template: java/app-complete-build.yml@templates - parameters: - artifactName: dapla-dlp-pseudo-service - gcrNamespace: dapla - mavenContainer: maven:3.8-openjdk-17 - checkStyleRunAnalysis: true - pmdRunAnalysis: true - sonarQubeRunAnalysis: false - sonarCloud: bipSonarCloud-Dapla diff --git a/cloudbuild.yml b/cloudbuild.yml deleted file mode 100644 index 7b453f5..0000000 --- a/cloudbuild.yml +++ /dev/null @@ -1,21 +0,0 @@ - -steps: - - # Build jar file - - name: 'gcr.io/cloud-builders/mvn' - entrypoint: 'mvn' - args: ['package','-Dmaven.test.skip=true', '-P ssb-bip'] - - # Build Docker image - - name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', - 'europe-north1-docker.pkg.dev/artifact-registry-14da/ssb-docker/ssb/statistikktjenester/automation/dapla-dlp-pseudo', - '.'] - - # Push Docker image to the ssb-docker Artifact Registry - - name: 'gcr.io/cloud-builders/docker' - args: ['push', - 'europe-north1-docker.pkg.dev/artifact-registry-14da/ssb-docker/ssb/statistikktjenester/automation/dapla-dlp-pseudo'] - -options: - logging: CLOUD_LOGGING_ONLY diff --git a/pom.xml b/pom.xml index 39eaa43..1be9710 100644 --- a/pom.xml +++ b/pom.xml @@ -1,9 +1,10 @@ 4.0.0 - no.ssb.dlp.pseudo.service + no.ssb.dapla.dlp.pseudo dapla-dlp-pseudo-service 3.0.0-SNAPSHOT + dapla-dlp-pseudo-service io.micronaut @@ -17,18 +18,21 @@ ${jdk.version} UTF-8 UTF-8 - artifactregistry://europe-north1-maven.pkg.dev/artifact-registry-5n/dapla-pseudo-maven/ + artifactregistry://europe-north1-maven.pkg.dev/artifact-registry-5n/dapla-pseudo-maven statisticsnorway/dapla-dlp-pseudo-service + no.ssb.dlp.pseudo.service.Application - 2.1.4 + 2.2.1 2.0.0 3.8.7 4.6.0 3.8.1 1.1.0 1.0.1 - no.ssb.dlp.pseudo.service.Application + 3.6.2 + 3.3.1 3.2.2 + 3.3.0 3.5.1 3.0.1 3.2.2 @@ -44,33 +48,20 @@ - artifact-registry - ${artifact-registry.url} - - true - - - false - + central + https://repo.maven.apache.org/maven2 - artifact-registry-snapshot + artifact-registry ${artifact-registry.url} - false + true + always true - - central - https://repo.maven.apache.org/maven2 - - - jcenter.bintray.com - https://jcenter.bintray.com - @@ -246,43 +237,139 @@ org.apache.maven.plugins - maven-shade-plugin - ${maven-shade-plugin.version} + maven-resources-plugin + ${maven-resources-plugin.version} + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} - default-shade + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + ${jdk.version} + + + org.projectlombok + lombok + ${lombok.version} + + + io.micronaut + micronaut-inject-java + ${micronaut.version} + + + io.micronaut + micronaut-validation + ${micronaut.version} + + + io.micronaut.security + micronaut-security-annotations + ${micronaut.security.version} + + + io.micronaut.openapi + micronaut-openapi + ${micronaut.openapi.version} + + + com.google.auto.service + auto-service + ${auto-service.version} + + + + + + test-compile + + testCompile + - false - - - ${exec.mainClass} - - - - - - *:* - - module-info.class - META-INF/*.MF - META-INF/*.txt - META-INF/LICENSE - META-INF/DEPENDENCIES - META-INF/NOTICE - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - + + + org.projectlombok + lombok + ${lombok.version} + + + io.micronaut + micronaut-inject-java + ${micronaut.version} + + + io.micronaut + micronaut-validation + ${micronaut.version} + + + io.micronaut.security + micronaut-security-annotations + ${micronaut.security.version} + + - + + org.apache.maven.plugins + maven-release-plugin + ${maven-release-plugin.version} + + @{project.version} + + + + org.apache.maven.plugins + maven-shade-plugin + ${maven-shade-plugin.version} + + + default-shade + + false + + + ${exec.mainClass} + + + + + + *:* + + module-info.class + META-INF/*.MF + META-INF/*.txt + META-INF/LICENSE + META-INF/DEPENDENCIES + META-INF/NOTICE + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + - io.micronaut.build @@ -291,15 +378,16 @@ org.apache.maven.plugins - maven-shade-plugin + maven-compiler-plugin org.apache.maven.plugins - maven-release-plugin - ${maven-release-plugin.version} - - @{project.version} - + maven-source-plugin + + + + org.apache.maven.plugins + maven-shade-plugin org.apache.maven.plugins @@ -325,77 +413,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - - ${jdk.version} - - - org.projectlombok - lombok - ${lombok.version} - - - io.micronaut - micronaut-inject-java - ${micronaut.version} - - - io.micronaut - micronaut-validation - ${micronaut.version} - - - io.micronaut.security - micronaut-security-annotations - ${micronaut.security.version} - - - io.micronaut.openapi - micronaut-openapi - ${micronaut.openapi.version} - - - com.google.auto.service - auto-service - ${auto-service.version} - - - - - - test-compile - - testCompile - - - - - org.projectlombok - lombok - ${lombok.version} - - - io.micronaut - micronaut-inject-java - ${micronaut.version} - - - io.micronaut - micronaut-validation - ${micronaut.version} - - - io.micronaut.security - micronaut-security-annotations - ${micronaut.security.version} - - - - - - pl.project13.maven git-commit-id-plugin diff --git a/src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoController.java b/src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoController.java index b02d890..dc9509d 100644 --- a/src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoController.java +++ b/src/main/java/no/ssb/dlp/pseudo/service/pseudo/PseudoController.java @@ -100,7 +100,7 @@ public HttpResponse pseudonymizeField(@Schema(implementation = PseudoF @Operation(summary = "Pseudonymize file", description = """ Pseudonymize a file (JSON or CSV - or a zip with potentially multiple such files) by uploading the file. - Choose between streaming the pseudonymized result back, or storing it as a file in GCS (by providing a `targetUri`). + The pseudonymized result will be streamed back. Notice that you can specify the `targetContentType` if you want to convert to either of the supported file formats. E.g. your source could be a CSV file and the result could be a JSON file. @@ -146,8 +146,8 @@ Depseudonymize a file (JSON or CSV - or a zip with potentially multiple such fil Notice that only certain whitelisted users can depseudonymize data. - Choose between streaming the result back, or storing it as a file in GCS (by providing a `targetUri`). - + The pseudonymized result will be streamed back. + Notice that you can specify the `targetContentType` if you want to convert to either of the supported file formats. E.g. your source could be a CSV file and the result could be a JSON file. @@ -194,7 +194,7 @@ public HttpResponse depseudonymizeFile(@Schema(implementation = Pseudo Repseudonymize a file (JSON or CSV - or a zip with potentially multiple such files) by uploading the file. Repseudonymization is done by first applying depseudonuymization and then pseudonymization to fields of the file. - Choose between streaming the result back, or storing it as a file in GCS (by providing a `targetUri`). + The pseudonymized result will be streamed back. Notice that you can specify the `targetContentType` if you want to convert to either of the supported file formats. E.g. your source could be a CSV file and the result could be a JSON file.