Add support for using a HTTPS catalog URL for arbitrary catalogs

gitlab/README.md

@@ -56,14 +56,30 @@ spec:
           masked: true
 ```
 
-### External cluster catalog
+### External cluster catalog via SSH
 
-If the cluster catalog is not hosted on the same GitLab instance as the tenant repo, you can specify an SSH key which has access to the cluster catalog and the relevant known hosts entry via CI/CD variables on the tenant repo:
+If the cluster catalog is hosted externally and can be cloned via SSH, you can specify an SSH key which has access to the cluster catalog and the relevant known hosts entry via CI/CD variables on the tenant repo:
 
 1. Create a CI/CD variable named `SSH_PRIVATE_KEY` containing the SSH private key.
 1. Create a CI/CD varaible named `SSH_KNOWN_HOSTS` containing the know hosts entry.
 1. (optional) Create a CI/CD variable named `SSH_CONFIG` containing any required SSH configuration.
 
+### External cluster catalog via HTTPS
+
+If the cluster catalog is hosted externally and must be cloned via HTTPS, you can configure HTTPS credentials via CI/CD variables on the tenant repo:
+
+1. Create a CI/CD variable named `ACCESS_USER_CLUSTERNAME` where `CLUSTERNAME` is the Project Syn ID of the cluster. 
+   Set this variable's value to the username used to access the catalog repo.
+1. Create a CI/CD variable named `ACCESS_TOKEN_CLUSTERNAME` where `CLUSTERNAME` is the Project Syn ID of the cluster.
+   Set this variable's value to the password or token used to access the catalog repo.
+
+> [!NOTE]
+> To make this work, the Project Syn cluster must be configured to provide its `catalogURL` with a `https://` prefix.
+
+> [!TIP]
+> The variable `ACCESS_USER_CLUSTERNAME` is optional.
+> If it's not provided, the CI pipeline will fallback to username `token`.
+
 ### Test new pipeline generation image
 
 The image used to generate the compile and deploy pipelines can be adjusted by setting the following variables.

gitlab/commodore-compile.jsonnet

@@ -21,21 +21,38 @@ local cpu_requests = to_array('cpu_requests');
 
 local gitInsteadOf(cluster) =
   local cluster_access_token = '${ACCESS_TOKEN_%s}' % std.strReplace(cluster, '-', '_');
+  local cluster_access_user = '${ACCESS_USER_%s:-token}' % std.strReplace(cluster, '-', '_');
   local cluster_repo = cluster_catalog_urls[cluster];
   local ssh_gitlab = 'ssh://git@%s/' % gitlab_ssh_host;
   local catalog_path = if std.startsWith(cluster_repo, ssh_gitlab) then
     // prefix ssh://git@<host> 0 == ssh, 1 == '', 2 == <host>
     std.join('/', std.split(cluster_repo, '/')[3:]);
 
+  local https_catalog = if std.startsWith(cluster_repo, 'https://') then
+    std.substr(cluster_repo, std.length('https://'), std.length(cluster_repo));
+
   local catalogInsteadOf =
     if catalog_path != null then
+      // set an insteadOf which injects the access token for catalog repos
+      // that have an ssh catalog URL and which are hosted on the local
+      // GitLab.
       [
         'git config --global url."https://gitlab-ci-token:%(access_token)s@%(gitlab_fqdn)s/%(catalog_path)s".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}/%(catalog_path)s' % {
           access_token: cluster_access_token,
           catalog_path: catalog_path,
           gitlab_fqdn: gitlab_fqdn,
         },
       ]
+    else if https_catalog != null then
+      // set an insteadOf which injects credentials if we have a catalog URL
+      // that's already HTTPS in Lieutenant.
+      [
+        'git config --global url."https://%(catalog_user)s:%(access_token)s@%(https_catalog)s".insteadOf https://%(https_catalog)s' % {
+          catalog_user: cluster_access_user,
+          access_token: cluster_access_token,
+          https_catalog: https_catalog,
+        },
+      ]
     else
       [];
 

gitlab/tests/external-catalog.env

@@ -1,2 +1,2 @@
 CLUSTERS="c-cluster-id-1234 c-cluster-id-5678 c-cluster-id-1111"
-CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://user:pass@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git5"
+CLUSTER_CATALOG_URLS="c-cluster-id-1234=ssh://git@git.vshn.net/cluster-catalogs/c-cluster-id-1234.git c-cluster-id-5678=ssh://git@git.example.com/cluster-catalogs/c-cluster-id-5678.git c-cluster-id-1111=https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git"

gitlab/tests/golden/external-catalog.yml

@@ -21,6 +21,7 @@
       ],
       "script": [
          "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git",
          "/usr/local/bin/entrypoint.sh commodore catalog compile --tenant-repo-revision-override $CI_COMMIT_SHA c-cluster-id-1111",
          "(cd catalog/ && git --no-pager diff --staged --output ../diff.txt)"
       ],
@@ -47,6 +48,7 @@
       ],
       "script": [
          "git config --global url.\"https://gitlab-ci-token:${CI_JOB_TOKEN}@git.vshn.net:80\".insteadOf ssh://git@${CI_SERVER_SHELL_SSH_HOST}",
+         "git config --global url.\"https://${ACCESS_USER_c_cluster_id_1111:-token}:${ACCESS_TOKEN_c_cluster_id_1111}@git.vshn.net/cluster-catalogs/c-cluster-id-1111.git\".insteadOf https://git.vshn.net/cluster-catalogs/c-cluster-id-1111.git",
          "/usr/local/bin/entrypoint.sh commodore catalog compile --push c-cluster-id-1111"
       ],
       "stage": "deploy",