Merge pull request #2 from projectsyn/feat/generator-golden-tests

Add golden tests for GitLab job generator Jsonnet script
projectsyn · Jul 26, 2024 · 337e1b3 · 337e1b3
2 parents 14ba118 + d4dd4fd
commit 337e1b3
Show file tree

Hide file tree

Showing 11 changed files with 499 additions and 0 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -31,3 +31,27 @@ jobs:
           pattern: |
             *.sh
             step-*
+
+  test_gitlab_generator_script_discover_cases:
+    name: Discover golden test cases
+    runs-on: ubuntu-latest
+    outputs:
+      instances: ${{ steps.instances.outputs.instances }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Find test cases
+        id: instances
+        run: |
+          echo "instances=$(make -sC gitlab list_test_instances)" >> "$GITHUB_OUTPUT"
+
+  test_gitlab_generator_script:
+    needs: test_gitlab_generator_script_discover_cases
+    strategy:
+      matrix:
+        instance: ${{ fromJSON(needs.test_gitlab_generator_script_discover_cases.outputs.instances) }}
+    runs-on: ubuntu-latest
+    name: 'Golden test: ${{ matrix.instance }}'
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          make -C gitlab golden-diff -e instance=${{ matrix.instance }}
diff --git a/.yamllint.yml b/.yamllint.yml
@@ -1,5 +1,7 @@
 ---
 extends: default
+ignore:
+  - /gitlab/tests/golden/
 rules:
   line-length: disable
   trailing-spaces:

diff --git a/gitlab/Makefile b/gitlab/Makefile
@@ -12,3 +12,31 @@ jsonnetfmt_check:
 jsonnetfmt: JSONNET_ENTRYPOINT=jsonnetfmt
 jsonnetfmt:
 	$(JSONNET_DOCKER) --in-place --pad-arrays -- *.jsonnet
+
+.PHONY: gen-golden
+gen-golden:
+	@rm -f tests/golden/$(instance).yml
+	$(JSONNET_DOCKER) tests/run-instance.sh $(instance).env > tests/golden/$(instance).yml
+
+.PHONY: golden-diff
+golden-diff:
+	@mkdir -p /tmp/commodore-compile-pipelines
+	$(JSONNET_DOCKER) tests/run-instance.sh $(instance).env > /tmp/commodore-compile-pipelines/$(instance).yml
+	@git diff --exit-code --minimal --no-index -- tests/golden/$(instance).yml /tmp/commodore-compile-pipelines/$(instance).yml
+
+.PHONY: golden-diff-all
+golden-diff-all: recursive_target=golden-diff
+golden-diff-all: $(test_instances)
+
+.PHONY: gen-golden-all
+gen-golden-all: recursive_target=gen-golden
+gen-golden-all: $(test_instances)
+
+.PHONY: $(test_instances)
+$(test_instances):
+	$(MAKE) $(recursive_target) -e instance=$(basename $(@F))
+
+.PHONY: list_test_instances
+list_test_instances: JSONNET_ENTRYPOINT=jsonnet
+list_test_instances:
+	$(JSONNET_DOCKER) --ext-str instances="$(basename $(notdir $(test_instances)))" -e 'std.split(std.extVar("instances"), " ")' | jq -c
diff --git a/gitlab/Makefile.vars.mk b/gitlab/Makefile.vars.mk
@@ -10,3 +10,6 @@ DOCKER_ARGS ?= run --rm -u "$$(id -u):$$(id -g)" --userns=$(DOCKER_USERNS) -w /w
 JSONNET_IMAGE      ?= docker.io/bitnami/jsonnet:latest
 JSONNET_ENTRYPOINT ?= bash
 JSONNET_DOCKER     ?= $(DOCKER_CMD) $(DOCKER_ARGS) -v "$${PWD}:/work" --entrypoint=$(JSONNET_ENTRYPOINT) $(JSONNET_IMAGE)
+
+test_instances=$(shell find tests/ -maxdepth 1 -name '*.env')
+instance=default
diff --git a/gitlab/tests/default.env b/gitlab/tests/default.env
@@ -0,0 +1,2 @@
+CLUSTERS="c-cluster-id-1234"
+CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git"
diff --git a/gitlab/tests/external-catalog.env b/gitlab/tests/external-catalog.env
@@ -0,0 +1,2 @@
+CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-1111"
+CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://user:pass@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5"
diff --git a/gitlab/tests/golden/default.yml b/gitlab/tests/golden/default.yml
@@ -0,0 +1,61 @@
+{
+   "c-cluster-id-1234_compile": {
+      "artifacts": {
+         "expire_in": "1 week",
+         "paths": [
+            "diff.txt"
+         ]
+      },
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1234}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1234.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1234.git",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1234",
+         "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
+      ],
+      "stage": "build",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-1234_deploy": {
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1234}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1234.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1234.git",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1234"
+      ],
+      "stage": "deploy",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   }
+}
diff --git a/gitlab/tests/golden/external-catalog.yml b/gitlab/tests/golden/external-catalog.yml
@@ -0,0 +1,175 @@
+{
+   "c-cluster-id-1111_compile": {
+      "artifacts": {
+         "expire_in": "1 week",
+         "paths": [
+            "diff.txt"
+         ]
+      },
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1111",
+         "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
+      ],
+      "stage": "build",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-1111_deploy": {
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1111"
+      ],
+      "stage": "deploy",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-1234_compile": {
+      "artifacts": {
+         "expire_in": "1 week",
+         "paths": [
+            "diff.txt"
+         ]
+      },
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1234}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1234.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1234.git",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1234",
+         "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
+      ],
+      "stage": "build",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-1234_deploy": {
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://gitlab-ci-token:${ACCESS_TOKEN_c_cluster_id_1234}@git.vshn.net:80/cluster-catalogs/c-cluster-id-1234.git\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/cluster-catalogs/c-cluster-id-1234.git",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1234"
+      ],
+      "stage": "deploy",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-5678_compile": {
+      "artifacts": {
+         "expire_in": "1 week",
+         "paths": [
+            "diff.txt"
+         ]
+      },
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME != $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-5678",
+         "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
+      ],
+      "stage": "build",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   },
+   "c-cluster-id-5678_deploy": {
+      "before_script": [
+         "install --directory --mode=0700 ~/.ssh",
+         "echo \"$SSH_KNOWN_HOSTS\" >> ~/.ssh/known_hosts",
+         "echo \"$SSH_CONFIG\" >> ~/.ssh/config"
+      ],
+      "image": {
+         "name": "docker.io/projectsyn/commodore:v1.22.1"
+      },
+      "rules": [
+         {
+            "if": "$CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH"
+         }
+      ],
+      "script": [
+         "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-5678"
+      ],
+      "stage": "deploy",
+      "variables": {
+         "KUBERNETES_CPU_LIMIT": "2",
+         "KUBERNETES_CPU_REQUEST": "800m",
+         "KUBERNETES_MEMORY_LIMIT": "2Gi"
+      }
+   }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		CLUSTERS="c-cluster-id-1234"
		CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git"
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-1111"
		CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://user:pass@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5"