From e6f1548b784acacf293db590b79e4f538ea14444 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 17 Dec 2024 11:15:56 +0100 Subject: [PATCH 1/3] Add support for using a HTTPS catalog URL for arbitrary catalogs --- gitlab/commodore-compile.jsonnet | 17 +++++++++++++++++ gitlab/tests/external-catalog.env | 2 +- gitlab/tests/golden/external-catalog.yml | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/gitlab/commodore-compile.jsonnet b/gitlab/commodore-compile.jsonnet index 5ce2bb2..0ccc732 100644 --- a/gitlab/commodore-compile.jsonnet +++ b/gitlab/commodore-compile.jsonnet @@ -21,14 +21,21 @@ local cpu_requests = to_array('cpu_requests'); local gitInsteadOf(cluster) = local cluster_access_token = '${ACCESS_TOKEN_%s}' % std.strReplace(cluster, '-', '_'); + local cluster_access_user = '${ACCESS_USER_%s:-token}' % std.strReplace(cluster, '-', '_'); local cluster_repo = cluster_catalog_urls[cluster]; local ssh_gitlab = 'ssh://git@%s/' % gitlab_ssh_host; local catalog_path = if std.startsWith(cluster_repo, ssh_gitlab) then // prefix ssh://git@ 0 == ssh, 1 == '', 2 == std.join('/', std.split(cluster_repo, '/')[3:]); + local https_catalog = if std.startsWith(cluster_repo, 'https://') then + std.substr(cluster_repo, std.length('https://'), std.length(cluster_repo)); + local catalogInsteadOf = if catalog_path != null then + // set an insteadOf which injects the access token for catalog repos + // that have an ssh catalog URL and which are hosted on the local + // GitLab. [ 'git config --global url."https://gitlab-ci-token:%(access_token)s@%(gitlab_fqdn)s/%(catalog_path)s".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/%(catalog_path)s' % { access_token: cluster_access_token, @@ -36,6 +43,16 @@ local gitInsteadOf(cluster) = gitlab_fqdn: gitlab_fqdn, }, ] + else if https_catalog != null then + // set an insteadOf which injects credentials if we have a catalog URL + // that's already HTTPS in Lieutenant. + [ + 'git config --global url."https://%(catalog_user)s:%(access_token)s@%(https_catalog)s".insteadOf https://%(https_catalog)s' % { + catalog_user: cluster_access_user, + access_token: cluster_access_token, + https_catalog: https_catalog, + }, + ] else []; diff --git a/gitlab/tests/external-catalog.env b/gitlab/tests/external-catalog.env index 9c842bc..309f7ec 100644 --- a/gitlab/tests/external-catalog.env +++ b/gitlab/tests/external-catalog.env @@ -1,2 +1,2 @@ CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-1111" -CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://user:pass@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5" +CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5" diff --git a/gitlab/tests/golden/external-catalog.yml b/gitlab/tests/golden/external-catalog.yml index ed71d77..e404ffa 100644 --- a/gitlab/tests/golden/external-catalog.yml +++ b/gitlab/tests/golden/external-catalog.yml @@ -21,6 +21,7 @@ ], "script": [ "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}", + "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5", "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1111", "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)" ], @@ -47,6 +48,7 @@ ], "script": [ "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}", + "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5", "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1111" ], "stage": "deploy", From 7f902e39f0675816704976b6f5107bfa332c99ec Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 17 Dec 2024 11:55:51 +0100 Subject: [PATCH 2/3] Update README --- gitlab/README.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/gitlab/README.md b/gitlab/README.md index bb84e55..f4de4b1 100644 --- a/gitlab/README.md +++ b/gitlab/README.md @@ -56,14 +56,30 @@ spec: masked: true ``` -### External cluster catalog +### External cluster catalog via SSH -If the cluster catalog is not hosted on the same GitLab instance as the tenant repo, you can specify an SSH key which has access to the cluster catalog and the relevant known hosts entry via CI/CD variables on the tenant repo: +If the cluster catalog is hosted externally and can be cloned via SSH, you can specify an SSH key which has access to the cluster catalog and the relevant known hosts entry via CI/CD variables on the tenant repo: 1. Create a CI/CD variable named `SSH_PRIVATE_KEY` containing the SSH private key. 1. Create a CI/CD varaible named `SSH_KNOWN_HOSTS` containing the know hosts entry. 1. (optional) Create a CI/CD variable named `SSH_CONFIG` containing any required SSH configuration. +### External cluster catalog via HTTPS + +If the cluster catalog is hosted externally and must be cloned via HTTPS, you can configure HTTPS credentials via CI/CD variables on the tenant repo: + +1. Create a CI/CD variable named `ACCESS_USER_CLUSTERNAME` where `CLUSTERNAME` is the Project Syn ID of the cluster. + Set this variable's value to the username used to access the catalog repo. +1. Create a CI/CD variable named `ACCESS_TOKEN_CLUSTERNAME` where `CLUSTERNAME` is the Project Syn ID of the cluster. + Set this variable's value to the password or token used to access the catalog repo. + +> [!NOTE] +> To make this work, the Project Syn cluster must be configured to provide its `catalogURL` with a `https://` prefix. + +> [!TIP] +> The variable `ACCESS_USER_CLUSTERNAME` is optional. +> If it's not provided, the CI pipeline will fallback to username `token`. + ### Test new pipeline generation image The image used to generate the compile and deploy pipelines can be adjusted by setting the following variables. From b92474038ec017b681f21f093fbe5bbbed808251 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 18 Dec 2024 11:26:09 +0100 Subject: [PATCH 3/3] Fix typo in external catalog test case --- gitlab/tests/external-catalog.env | 2 +- gitlab/tests/golden/external-catalog.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gitlab/tests/external-catalog.env b/gitlab/tests/external-catalog.env index 309f7ec..d8729e1 100644 --- a/gitlab/tests/external-catalog.env +++ b/gitlab/tests/external-catalog.env @@ -1,2 +1,2 @@ CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-1111" -CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5" +CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git" diff --git a/gitlab/tests/golden/external-catalog.yml b/gitlab/tests/golden/external-catalog.yml index e404ffa..dfe5f1c 100644 --- a/gitlab/tests/golden/external-catalog.yml +++ b/gitlab/tests/golden/external-catalog.yml @@ -21,7 +21,7 @@ ], "script": [ "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}", - "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5", + "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git", "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1111", "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)" ], @@ -48,7 +48,7 @@ ], "script": [ "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}", - "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5", + "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git", "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1111" ], "stage": "deploy",