diff --git a/api/v1alpha1/gitrepo_types.go b/api/v1alpha1/gitrepo_types.go index 9e336894..bd170ea9 100644 --- a/api/v1alpha1/gitrepo_types.go +++ b/api/v1alpha1/gitrepo_types.go @@ -87,6 +87,60 @@ type GitRepoTemplate struct { // Adopt: will create a new external resource or will adopt and manage an already existing resource // +kubebuilder:validation:Enum=Create;Adopt CreationPolicy CreationPolicy `json:"creationPolicy,omitempty"` + // AccessToken contains configuration for storing an access token in a secret. + // If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + // The token is stored under the key "token". + // In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + AccessToken AccessToken `json:"accessToken,omitempty"` + // CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + // + // The variables are not expanded like PodSpec environment variables. + CIVariables []EnvVar `json:"ciVariables,omitempty"` +} + +type AccessToken struct { + // SecretRef references the secret the access token is stored in + SecretRef string `json:"secretRef,omitempty"` +} + +// EnvVar represents an environment added to the CI system of the Git repository. +type EnvVar struct { + // Name of the environment variable + // +required + Name string `json:"name"` + // Value of the environment variable + // +optional + Value string `json:"value,omitempty"` + + // ValueFrom is a reference to an object that contains the value of the environment variable + // +optional + ValueFrom *EnvVarSource `json:"valueFrom,omitempty"` + + // GitlabOptions contains additional options for GitLab CI variables + // +optional + GitlabOptions EnvVarGitlabOptions `json:"gitlabOptions,omitempty"` +} + +type EnvVarGitlabOptions struct { + // Description is a description of the CI variable. + // +optional + Description string `json:"description,omitempty"` + // Protected will expose the variable only in protected branches and tags. + // +optional + Protected bool `json:"protected,omitempty"` + // Masked will mask the variable in the job logs. + // +optional + Masked bool `json:"masked,omitempty"` + // Raw will prevent the variable from being expanded. + // +optional + Raw bool `json:"raw,omitempty"` +} + +// EnvVarSource represents a source for the value of an EnvVar. +type EnvVarSource struct { + // Selects a key of a secret in the pod's namespace + // +optional + SecretKeyRef *corev1.SecretKeySelector `json:"secretKeyRef,omitempty"` } // DeployKey defines an SSH key to be used for git operations. @@ -110,6 +164,8 @@ type GitRepoStatus struct { URL string `json:"url,omitempty"` // SSH HostKeys of the git server HostKeys string `json:"hostKeys,omitempty"` + // LastAppliedCIVariables contains the last applied CI variables as a json string + LastAppliedCIVariables string `json:"lastAppliedCIVariables,omitempty"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9b14b693..a47fc4ed 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5,9 +5,25 @@ package v1alpha1 import ( + "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AccessToken) DeepCopyInto(out *AccessToken) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AccessToken. +func (in *AccessToken) DeepCopy() *AccessToken { + if in == nil { + return nil + } + out := new(AccessToken) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BootstrapToken) DeepCopyInto(out *BootstrapToken) { *out = *in @@ -224,6 +240,62 @@ func (in *DeployKey) DeepCopy() *DeployKey { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvVar) DeepCopyInto(out *EnvVar) { + *out = *in + if in.ValueFrom != nil { + in, out := &in.ValueFrom, &out.ValueFrom + *out = new(EnvVarSource) + (*in).DeepCopyInto(*out) + } + out.GitlabOptions = in.GitlabOptions +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvVar. +func (in *EnvVar) DeepCopy() *EnvVar { + if in == nil { + return nil + } + out := new(EnvVar) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvVarGitlabOptions) DeepCopyInto(out *EnvVarGitlabOptions) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvVarGitlabOptions. +func (in *EnvVarGitlabOptions) DeepCopy() *EnvVarGitlabOptions { + if in == nil { + return nil + } + out := new(EnvVarGitlabOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EnvVarSource) DeepCopyInto(out *EnvVarSource) { + *out = *in + if in.SecretKeyRef != nil { + in, out := &in.SecretKeyRef, &out.SecretKeyRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EnvVarSource. +func (in *EnvVarSource) DeepCopy() *EnvVarSource { + if in == nil { + return nil + } + out := new(EnvVarSource) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in Facts) DeepCopyInto(out *Facts) { { @@ -359,6 +431,14 @@ func (in *GitRepoTemplate) DeepCopyInto(out *GitRepoTemplate) { (*out)[key] = val } } + out.AccessToken = in.AccessToken + if in.CIVariables != nil { + in, out := &in.CIVariables, &out.CIVariables + *out = make([]EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GitRepoTemplate. diff --git a/config/crd/bases/syn.tools_clusters.yaml b/config/crd/bases/syn.tools_clusters.yaml index afadc4e2..7535b9d6 100644 --- a/config/crd/bases/syn.tools_clusters.yaml +++ b/config/crd/bases/syn.tools_clusters.yaml @@ -85,6 +85,18 @@ spec: gitRepoTemplate: description: GitRepoTemplate template for managing the GitRepo object. properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access token + is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -99,6 +111,79 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the CI + system of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options for + GitLab CI variables + properties: + description: + description: Description is a description of the CI + variable. + type: string + masked: + description: Masked will mask the variable in the job + logs. + type: boolean + protected: + description: Protected will expose the variable only + in protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from being + expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that + contains the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. diff --git a/config/crd/bases/syn.tools_gitrepos.yaml b/config/crd/bases/syn.tools_gitrepos.yaml index 31045d0f..c829b7ab 100644 --- a/config/crd/bases/syn.tools_gitrepos.yaml +++ b/config/crd/bases/syn.tools_gitrepos.yaml @@ -53,6 +53,18 @@ spec: spec: description: GitRepoSpec defines the desired state of GitRepo properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access token + is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -67,6 +79,75 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the CI system + of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options for GitLab + CI variables + properties: + description: + description: Description is a description of the CI variable. + type: string + masked: + description: Masked will mask the variable in the job logs. + type: boolean + protected: + description: Protected will expose the variable only in + protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from being expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that contains + the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. @@ -152,6 +233,10 @@ spec: hostKeys: description: SSH HostKeys of the git server type: string + lastAppliedCIVariables: + description: LastAppliedCIVariables contains the last applied CI variables + as a json string + type: string phase: description: |- Updated by Operator with current phase. The GitPhase enum will be used for application logic diff --git a/config/crd/bases/syn.tools_tenants.yaml b/config/crd/bases/syn.tools_tenants.yaml index 0a18964d..b0987b39 100644 --- a/config/crd/bases/syn.tools_tenants.yaml +++ b/config/crd/bases/syn.tools_tenants.yaml @@ -90,6 +90,18 @@ spec: description: GitRepoTemplate template for managing the GitRepo object. properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access + token is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -104,6 +116,79 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the + CI system of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options + for GitLab CI variables + properties: + description: + description: Description is a description of the + CI variable. + type: string + masked: + description: Masked will mask the variable in the + job logs. + type: boolean + protected: + description: Protected will expose the variable + only in protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from + being expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that + contains the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. @@ -238,6 +323,18 @@ spec: description: GitRepoTemplate Template for managing the GitRepo object. If not set, no GitRepo object will be created. properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access token + is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -252,6 +349,79 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the CI + system of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options for + GitLab CI variables + properties: + description: + description: Description is a description of the CI + variable. + type: string + masked: + description: Masked will mask the variable in the job + logs. + type: boolean + protected: + description: Protected will expose the variable only + in protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from being + expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that + contains the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. diff --git a/config/crd/bases/syn.tools_tenanttemplates.yaml b/config/crd/bases/syn.tools_tenanttemplates.yaml index 8058aa4f..56273db8 100644 --- a/config/crd/bases/syn.tools_tenanttemplates.yaml +++ b/config/crd/bases/syn.tools_tenanttemplates.yaml @@ -90,6 +90,18 @@ spec: description: GitRepoTemplate template for managing the GitRepo object. properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access + token is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -104,6 +116,79 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the + CI system of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options + for GitLab CI variables + properties: + description: + description: Description is a description of the + CI variable. + type: string + masked: + description: Masked will mask the variable in the + job logs. + type: boolean + protected: + description: Protected will expose the variable + only in protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from + being expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that + contains the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. @@ -238,6 +323,18 @@ spec: description: GitRepoTemplate Template for managing the GitRepo object. If not set, no GitRepo object will be created. properties: + accessToken: + description: |- + AccessToken contains configuration for storing an access token in a secret. + If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. + The token is stored under the key "token". + In the case of GitLab, this would be a Project Access Token with read-write access to the repository. + properties: + secretRef: + description: SecretRef references the secret the access token + is stored in + type: string + type: object apiSecretRef: description: APISecretRef reference to secret containing connection information @@ -252,6 +349,79 @@ spec: type: string type: object x-kubernetes-map-type: atomic + ciVariables: + description: |- + CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + + The variables are not expanded like PodSpec environment variables. + items: + description: EnvVar represents an environment added to the CI + system of the Git repository. + properties: + gitlabOptions: + description: GitlabOptions contains additional options for + GitLab CI variables + properties: + description: + description: Description is a description of the CI + variable. + type: string + masked: + description: Masked will mask the variable in the job + logs. + type: boolean + protected: + description: Protected will expose the variable only + in protected branches and tags. + type: boolean + raw: + description: Raw will prevent the variable from being + expanded. + type: boolean + type: object + name: + description: Name of the environment variable + type: string + value: + description: Value of the environment variable + type: string + valueFrom: + description: ValueFrom is a reference to an object that + contains the value of the environment variable + properties: + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array creationPolicy: description: |- CreationPolicy defines how the external resources should be treated upon CR creation. diff --git a/controllers/gitrepo/steps.go b/controllers/gitrepo/steps.go index 40a90371..b8c777e0 100644 --- a/controllers/gitrepo/steps.go +++ b/controllers/gitrepo/steps.go @@ -3,15 +3,25 @@ package gitrepo import ( "context" "fmt" + "time" "github.com/go-logr/logr" synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" + "go.uber.org/multierr" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/json" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/log" // Register Gitrepo implementation - DONOT REMOVE _ "github.com/projectsyn/lieutenant-operator/git" "github.com/projectsyn/lieutenant-operator/git/manager" "github.com/projectsyn/lieutenant-operator/pipeline" - "sigs.k8s.io/controller-runtime/pkg/client" ) func Steps(obj pipeline.Object, data *pipeline.Context) pipeline.Result { @@ -78,6 +88,14 @@ func steps(obj pipeline.Object, data *pipeline.Context, getGitClient gitClientFa return pipeline.Result{} } + if err := ensureAccessToken(data.Context, data.Client, instance, repo); err != nil { + return pipeline.Result{Err: handleRepoError(data.Context, fmt.Errorf("ensure access token: %w", err), instance, data.Client)} + } + + if err := ensureCIVariables(data.Context, data.Client, instance, repo); err != nil { + return pipeline.Result{Err: handleRepoError(data.Context, fmt.Errorf("ensure ci variables: %w", err), instance, data.Client)} + } + err = repo.CommitTemplateFiles() if err != nil { return pipeline.Result{Err: handleRepoError(data.Context, err, instance, data.Client)} @@ -114,3 +132,151 @@ func handleRepoError(ctx context.Context, err error, instance *synv1alpha1.GitRe } return err } + +const ( + LieutenantAccessTokenUIDAnnotation = "lieutenant.syn.tools/accessTokenUID" + LieutenantAccessTokenExpiresAtAnnotation = "lieutenant.syn.tools/accessTokenExpiresAt" +) + +// ensureAccessToken ensures that an up-to-date access token returned from the manager is stored in the referenced secret. +// It passes the UID of the previous access token to the manager to ensure that the same access token is returned if it has not expired. +func ensureAccessToken(ctx context.Context, cli client.Client, instance *synv1alpha1.GitRepo, repo manager.Repo) error { + name := instance.Spec.AccessToken.SecretRef + if name == "" { + return nil + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: instance.Namespace, + }, + } + op, err := controllerutil.CreateOrUpdate(ctx, cli, secret, func() error { + uid := secret.Annotations[LieutenantAccessTokenUIDAnnotation] + + pat, err := repo.EnsureProjectAccessToken(ctx, instance.GetName(), manager.EnsureProjectAccessTokenOptions{ + UID: uid, + }) + if err != nil { + return fmt.Errorf("error ensuring project access token: %w", err) + } + + if pat.Updated() { + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[LieutenantAccessTokenUIDAnnotation] = pat.UID + secret.Annotations[LieutenantAccessTokenExpiresAtAnnotation] = pat.ExpiresAt.Format(time.RFC3339) + secret.Data = map[string][]byte{ + "token": []byte(pat.Token), + } + } + + return controllerutil.SetControllerReference(instance, secret, cli.Scheme()) + }) + if err != nil { + return fmt.Errorf("error creating or updating access token secret: %w", err) + } + log.FromContext(ctx).Info("Reconciled secret", "secret", secret, "op", op) + + return nil +} + +// ensureCIVariables ensures that the CI variables are set on the repository. +// It calls the manager with the current variables from the CRD and a combination of the previous variables and the current variables as the managed variables. +func ensureCIVariables(ctx context.Context, cli client.Client, instance *synv1alpha1.GitRepo, repo manager.Repo) error { + var prevVars []synv1alpha1.EnvVar + if instance.Status.LastAppliedCIVariables != "" { + if err := json.Unmarshal([]byte(instance.Status.LastAppliedCIVariables), &prevVars); err != nil { + return fmt.Errorf("error unmarshalling previous ci variables: %w", err) + } + } + managedVars := sets.New[string]() + for _, v := range instance.Spec.CIVariables { + managedVars.Insert(v.Name) + } + for _, v := range prevVars { + managedVars.Insert(v.Name) + } + + vars := make([]manager.EnvVar, 0, len(instance.Spec.CIVariables)) + valueFromErrs := make([]error, 0, len(instance.Spec.CIVariables)) + for _, v := range instance.Spec.CIVariables { + val, err := valueFromEnvVar(ctx, cli, instance.Namespace, v) + if err != nil { + valueFromErrs = append(valueFromErrs, err) + continue + } + vars = append(vars, manager.EnvVar{ + Name: v.Name, + Value: val, + + GitlabOptions: manager.EnvVarGitlabOptions{ + Description: ptr.To(v.GitlabOptions.Description), + Protected: ptr.To(v.GitlabOptions.Protected), + Masked: ptr.To(v.GitlabOptions.Masked), + Raw: ptr.To(v.GitlabOptions.Raw), + }, + }) + } + if err := multierr.Combine(valueFromErrs...); err != nil { + return fmt.Errorf("error collecting values for env vars: %w", err) + } + + if err := repo.EnsureCIVariables(ctx, sets.List(managedVars), vars); err != nil { + return fmt.Errorf("error ensuring ci variables: %w", err) + } + + varsJSON, err := json.Marshal(instance.Spec.CIVariables) + if err != nil { + return fmt.Errorf("error marshalling ci variables: %w", err) + } + + instance.Status.LastAppliedCIVariables = string(varsJSON) + return nil +} + +// valueFromEnvVar returns the value of an envVar. It returns an error if the envVar is invalid or the value cannot be retrieved. +// EnvVars with both value and valueFrom are invalid. +// An envVar with no value and no valueFrom returns an empty string. +// If valueFrom is set but the secret reference is not valid, an error is returned. +// If the secret does not exist and the secretKeyRef is optional, an empty string is returned. Otherwise, an error is returned. +func valueFromEnvVar(ctx context.Context, cli client.Client, namespace string, envVar synv1alpha1.EnvVar) (string, error) { + l := log.FromContext(ctx).WithName("valueFromEnvVar") + + if envVar.Value != "" { + if envVar.ValueFrom != nil { + return "", fmt.Errorf("envVar %q has both value and valueFrom", envVar.Name) + } + return envVar.Value, nil + } + if envVar.ValueFrom == nil { + return "", nil + } + if envVar.ValueFrom.SecretKeyRef == nil { + return "", fmt.Errorf("envVar %q has no secretKeyRef", envVar.Name) + } + if envVar.ValueFrom.SecretKeyRef.Name == "" || envVar.ValueFrom.SecretKeyRef.Key == "" { + return "", fmt.Errorf("envVar %q has incomplete secretKeyRef", envVar.Name) + } + optional := ptr.Deref(envVar.ValueFrom.SecretKeyRef.Optional, false) + secret := &corev1.Secret{} + err := cli.Get(ctx, client.ObjectKey{Namespace: namespace, Name: envVar.ValueFrom.SecretKeyRef.Name}, secret) + if err != nil { + if apierrors.IsNotFound(err) && optional { + l.Info("secret not found but is optional, returning empty string", "secret", envVar.ValueFrom.SecretKeyRef.Name) + return "", nil + } + return "", fmt.Errorf("error getting secret %q: %w", envVar.ValueFrom.SecretKeyRef.Name, err) + } + val, ok := secret.Data[envVar.ValueFrom.SecretKeyRef.Key] + if !ok { + if optional { + l.Info("key not found but secret is optional, returning empty string", "key", envVar.ValueFrom.SecretKeyRef.Key, "secret", envVar.ValueFrom.SecretKeyRef.Name) + return "", nil + } + return "", fmt.Errorf("secret %q does not contain key %q", envVar.ValueFrom.SecretKeyRef.Name, envVar.ValueFrom.SecretKeyRef.Key) + } + return string(val), nil +} diff --git a/controllers/gitrepo/steps_test.go b/controllers/gitrepo/steps_test.go index 0c483953..6c173593 100644 --- a/controllers/gitrepo/steps_test.go +++ b/controllers/gitrepo/steps_test.go @@ -5,16 +5,19 @@ import ( "errors" "net/url" "testing" + "time" "github.com/go-logr/logr" + "github.com/go-logr/logr/testr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -201,7 +204,7 @@ func TestSteps(t *testing.T) { Context: context.TODO(), FinalizerName: "foo", Client: c, - Log: logr.Discard(), + Log: testr.New(t), Deleted: tc.deleted, } repoURL, err := url.Parse(tc.repoUrl) @@ -227,6 +230,63 @@ func TestSteps(t *testing.T) { } } +func TestSteps_AccessToken(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(synv1alpha1.AddToScheme(scheme)) + + repo := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "c-bar", + Namespace: "foo", + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{ + AccessToken: synv1alpha1.AccessToken{ + SecretRef: "buzz", + }, + }, + }, + } + + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(repo). + WithStatusSubresource(&synv1alpha1.GitRepo{}). + Build() + pContext := &pipeline.Context{ + Context: context.TODO(), + FinalizerName: "foo", + Client: c, + Log: testr.New(t), + } + fr := &fakeRepo{ + exists: true, + url: new(url.URL), + accessToken: manager.ProjectAccessToken{ + UID: "asdlfgkj", + Token: "token", + ExpiresAt: time.Now().Add(30 * 24 * time.Hour), + }, + } + gc := fakeGitClientFactory(fr) + res := steps(repo, pContext, gc) + assert.NoError(t, res.Err) + + var secret corev1.Secret + require.NoError(t, c.Get(pContext.Context, types.NamespacedName{Namespace: repo.Namespace, Name: repo.Spec.AccessToken.SecretRef}, &secret)) + assert.Equal(t, fr.accessToken.UID, secret.Annotations["lieutenant.syn.tools/accessTokenUID"]) + assert.Equal(t, fr.accessToken.Token, string(secret.Data["token"])) + assert.Equal(t, fr.accessToken.ExpiresAt.Format(time.RFC3339), secret.Annotations["lieutenant.syn.tools/accessTokenExpiresAt"]) + + oldToken := fr.accessToken.Token + fr.accessToken.Token = "" + res = steps(repo, pContext, gc) + assert.NoError(t, res.Err) + require.NoError(t, c.Get(pContext.Context, types.NamespacedName{Namespace: repo.Namespace, Name: repo.Spec.AccessToken.SecretRef}, &secret)) + assert.Equal(t, oldToken, string(secret.Data["token"])) +} + func TestStepsCreationFailure(t *testing.T) { scheme := runtime.NewScheme() utilruntime.Must(clientgoscheme.AddToScheme(scheme)) @@ -303,12 +363,164 @@ func TestStepsCreationFailure(t *testing.T) { } } +func TestSteps_CIVariables(t *testing.T) { + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(synv1alpha1.AddToScheme(scheme)) + + repo := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "c-bar", + Namespace: "foo", + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{ + CIVariables: []synv1alpha1.EnvVar{ + { + Name: "VALUE", + Value: "bar", + }, + { + Name: "EMPTY_VALUE", + }, + { + Name: "VALUE_FROM", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "qux-var", + }, + Key: "qux", + }, + }, + }, + { + Name: "OPTIONAL_VALUE", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "non-existing-secret", + }, + Key: "key", + Optional: ptr.To(true), + }, + }, + }, + { + Name: "OPTIONAL_VALUE_KEY_MISSING", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "qux-var", + }, + Key: "other-key", + Optional: ptr.To(true), + }, + }, + }, + }, + }, + }, + } + varNames := make([]string, 0, len(repo.Spec.CIVariables)) + for _, v := range repo.Spec.CIVariables { + varNames = append(varNames, v.Name) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "qux-var", + Namespace: "foo", + }, + Data: map[string][]byte{ + "qux": []byte("qux value"), + }, + } + + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(repo, secret). + WithStatusSubresource(&synv1alpha1.GitRepo{}). + Build() + pContext := &pipeline.Context{ + Context: context.TODO(), + FinalizerName: "foo", + Client: c, + Log: testr.New(t), + } + fr := &fakeRepo{ + exists: true, + url: new(url.URL), + } + gc := fakeGitClientFactory(fr) + res := steps(repo, pContext, gc) + assert.NoError(t, res.Err) + + require.Len(t, fr.ensureCIVariablesCalls, 1) + call := fr.ensureCIVariablesCalls[0] + assert.ElementsMatch(t, varNames, call.managed) + glo := manager.EnvVarGitlabOptions{ + Description: ptr.To(""), + Protected: ptr.To(false), + Masked: ptr.To(false), + Raw: ptr.To(false), + } + assert.ElementsMatch(t, []manager.EnvVar{ + { + Name: "VALUE", + Value: "bar", + + GitlabOptions: glo, + }, + { + Name: "EMPTY_VALUE", + + GitlabOptions: glo, + }, + { + Name: "VALUE_FROM", + Value: "qux value", + + GitlabOptions: glo, + }, + { + Name: "OPTIONAL_VALUE", + + GitlabOptions: glo, + }, + { + Name: "OPTIONAL_VALUE_KEY_MISSING", + + GitlabOptions: glo, + }, + }, call.vars) + + // remove first variable. Removed variable should still appear in the managed variables because it was managed before. + // it should not appear in the variables to be set. + repo.Spec.GitRepoTemplate.CIVariables = repo.Spec.GitRepoTemplate.CIVariables[1:] + res2 := steps(repo, pContext, gc) + assert.NoError(t, res2.Err) + require.Len(t, fr.ensureCIVariablesCalls, 2) + call = fr.ensureCIVariablesCalls[1] + assert.ElementsMatch(t, varNames, call.managed, "managed variables should be remembered from previous run") + callVarNames := make([]string, 0, len(call.vars)) + for _, v := range call.vars { + callVarNames = append(callVarNames, v.Name) + } + assert.ElementsMatch(t, varNames[1:], callVarNames) +} + func fakeGitClientFactory(r *fakeRepo) gitClientFactory { return func(ctx context.Context, instance *synv1alpha1.GitRepoTemplate, namespace string, reqLogger logr.Logger, client client.Client) (manager.Repo, string, error) { return r, "", nil } } +type ensureCIVariablesCall struct { + managed []string + vars []manager.EnvVar +} + type fakeRepo struct { url *url.URL @@ -322,6 +534,10 @@ type fakeRepo struct { failCreation bool failUpdate bool failCommit bool + + accessToken manager.ProjectAccessToken + + ensureCIVariablesCalls []ensureCIVariablesCall } func (r fakeRepo) Type() string { @@ -365,3 +581,13 @@ func (r *fakeRepo) CommitTemplateFiles() error { r.committed = true return nil } +func (r *fakeRepo) EnsureProjectAccessToken(ctx context.Context, name string, opts manager.EnsureProjectAccessTokenOptions) (manager.ProjectAccessToken, error) { + return r.accessToken, nil +} +func (r *fakeRepo) EnsureCIVariables(ctx context.Context, managed []string, vars []manager.EnvVar) error { + r.ensureCIVariablesCalls = append(r.ensureCIVariablesCalls, ensureCIVariablesCall{ + managed: managed, + vars: vars, + }) + return nil +} diff --git a/controllers/gitrepo/watchers/ci_var_secret.go b/controllers/gitrepo/watchers/ci_var_secret.go new file mode 100644 index 00000000..92f49fd0 --- /dev/null +++ b/controllers/gitrepo/watchers/ci_var_secret.go @@ -0,0 +1,62 @@ +package watchers + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" +) + +const ( + // GitRepoCIVariableValueFromSecretKeyRefNameIndex is the index name for the GitRepo objects that reference a secret by name. + GitRepoCIVariableValueFromSecretKeyRefNameIndex = "spec.ciVariables.valueFrom.secretKeyRef.name" +) + +// SecretGitRepoCIVariablesMapFunc returns a handler function that will return a list of reconcile.Requests for GitRepo objects +// that reference the secret in the given Secret object. +// It requires the field index GitRepoCIVariableValueFromSecretKeyRefNameIndex to be installed for the GitRepo objects. +func SecretGitRepoCIVariablesMapFunc(cli client.Client) func(ctx context.Context, o client.Object) []reconcile.Request { + return func(ctx context.Context, o client.Object) []reconcile.Request { + l := log.FromContext(ctx).WithName("SecretGitRepoCIVariablesMapFunc").WithValues("secret", o.GetName()) + + secret := o.(*corev1.Secret) + var gitRepos synv1alpha1.GitRepoList + if err := cli.List(ctx, &gitRepos, client.MatchingFields{ + GitRepoCIVariableValueFromSecretKeyRefNameIndex: secret.Name, + }, client.InNamespace(secret.GetNamespace())); err != nil { + l.Error(err, "unable to list GitRepos") + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, 0, len(gitRepos.Items)) + for _, gitRepo := range gitRepos.Items { + requests = append(requests, reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: gitRepo.Namespace, + Name: gitRepo.Name, + }, + }) + } + + return requests + } +} + +// GitRepoCIVariableValueFromSecretKeyRefNameIndexFunc is an index function for GitRepo objects. +// It indexes the names of the secrets that are referenced by the CIVariables of the GitRepo. +func GitRepoCIVariableValueFromSecretKeyRefNameIndexFunc(obj client.Object) []string { + gitRepo := obj.(*synv1alpha1.GitRepo) + values := make([]string, 0, len(gitRepo.Spec.CIVariables)) + for _, ciVariable := range gitRepo.Spec.CIVariables { + if ciVariable.ValueFrom != nil && + ciVariable.ValueFrom.SecretKeyRef != nil && + ciVariable.ValueFrom.SecretKeyRef.Name != "" { + values = append(values, ciVariable.ValueFrom.SecretKeyRef.Name) + } + } + return values +} diff --git a/controllers/gitrepo/watchers/ci_var_secret_test.go b/controllers/gitrepo/watchers/ci_var_secret_test.go new file mode 100644 index 00000000..69a3377e --- /dev/null +++ b/controllers/gitrepo/watchers/ci_var_secret_test.go @@ -0,0 +1,158 @@ +package watchers_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" + "github.com/projectsyn/lieutenant-operator/controllers/gitrepo/watchers" +) + +func TestIndexAndMapFunc(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(scheme)) + require.NoError(t, synv1alpha1.AddToScheme(scheme)) + + defaultNs := "test-namespace" + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "important-ci-stuff", + Namespace: defaultNs, + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + secret2 := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "important-ci-stuff-too", + Namespace: defaultNs, + }, + Data: map[string][]byte{ + "key": []byte("value"), + }, + } + + repoWithSecretRef := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-with-secret-ref", + Namespace: secret.GetNamespace(), + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{ + CIVariables: []synv1alpha1.EnvVar{ + { + Name: "KEY", + Value: "value", + }, + { + Name: "SECRET", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret.Name, + }, + Key: "key", + }, + }, + }, + { + Name: "SECRET2", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: secret2.Name, + }, + Key: "key", + }, + }, + }, + }, + }, + }, + } + repoWithOtherSecretRef := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-with-other-secret-ref", + Namespace: secret.GetNamespace(), + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{ + CIVariables: []synv1alpha1.EnvVar{ + { + Name: "KEY", + Value: "value", + }, + { + Name: "SECRET", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "other-secret", + }, + Key: "key", + }, + }, + }, + }, + }, + }, + } + repoWithSecretRefInOtherNs := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-with-secret-ref-in-other-ns", + Namespace: "other-namespace", + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{ + CIVariables: []synv1alpha1.EnvVar{ + { + Name: "KEY", + Value: "value", + }, + { + Name: "SECRET", + ValueFrom: &synv1alpha1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "other-secret", + }, + Key: "key", + }, + }, + }, + }, + }, + }, + } + repoWithoutRef := &synv1alpha1.GitRepo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-without-ref", + Namespace: defaultNs, + }, + Spec: synv1alpha1.GitRepoSpec{ + GitRepoTemplate: synv1alpha1.GitRepoTemplate{}, + }, + } + + c := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(secret, secret2, repoWithSecretRef, repoWithOtherSecretRef, repoWithSecretRefInOtherNs, repoWithoutRef). + WithIndex(&synv1alpha1.GitRepo{}, watchers.GitRepoCIVariableValueFromSecretKeyRefNameIndex, watchers.GitRepoCIVariableValueFromSecretKeyRefNameIndexFunc). + Build() + + requests := watchers.SecretGitRepoCIVariablesMapFunc(c)(context.Background(), secret) + require.Len(t, requests, 1) + require.Equal(t, repoWithSecretRef.Name, requests[0].Name) + requests = watchers.SecretGitRepoCIVariablesMapFunc(c)(context.Background(), secret2) + require.Len(t, requests, 1) + require.Equal(t, repoWithSecretRef.Name, requests[0].Name) +} diff --git a/controllers/gitrepo_controller.go b/controllers/gitrepo_controller.go index c0201bb3..c08651cf 100644 --- a/controllers/gitrepo_controller.go +++ b/controllers/gitrepo_controller.go @@ -2,17 +2,21 @@ package controllers import ( "context" + "fmt" - "github.com/projectsyn/lieutenant-operator/controllers/gitrepo" - "github.com/projectsyn/lieutenant-operator/pipeline" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" + "github.com/projectsyn/lieutenant-operator/controllers/gitrepo" + "github.com/projectsyn/lieutenant-operator/controllers/gitrepo/watchers" + "github.com/projectsyn/lieutenant-operator/pipeline" ) // GitRepoReconciler reconciles a GitRepo object @@ -70,8 +74,23 @@ func (r *GitRepoReconciler) Reconcile(ctx context.Context, request ctrl.Request) } // SetupWithManager sets up the controller with the Manager. -func (r *GitRepoReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *GitRepoReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + err := mgr.GetFieldIndexer().IndexField( + ctx, + &synv1alpha1.GitRepo{}, + watchers.GitRepoCIVariableValueFromSecretKeyRefNameIndex, + watchers.GitRepoCIVariableValueFromSecretKeyRefNameIndexFunc, + ) + if err != nil { + return fmt.Errorf("unable to create index for GitRepo: %w", err) + } + return ctrl.NewControllerManagedBy(mgr). For(&synv1alpha1.GitRepo{}). + Owns(&corev1.Secret{}). + Watches( + &corev1.Secret{}, + handler.EnqueueRequestsFromMapFunc(watchers.SecretGitRepoCIVariablesMapFunc(mgr.GetClient())), + ). Complete(r) } diff --git a/controllers/tenant_controller.go b/controllers/tenant_controller.go index 20c511a4..e4958b76 100644 --- a/controllers/tenant_controller.go +++ b/controllers/tenant_controller.go @@ -12,6 +12,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -82,5 +83,33 @@ func (r *TenantReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). + // Reconcile all tenants when a TenantTemplate changes to ensure that all tenants are up to date + Watches(&synv1alpha1.TenantTemplate{}, handler.EnqueueRequestsFromMapFunc(enqueueAllTenantsMapFunc(mgr.GetClient()))). Complete(r) } + +// enqueueAllTenantsMapFunc returns a function that lists all tenants in the namespace of the given object and returns a list of reconcile.Requests for all of them. +func enqueueAllTenantsMapFunc(cli client.Client) func(ctx context.Context, o client.Object) []reconcile.Request { + return func(ctx context.Context, o client.Object) []reconcile.Request { + l := log.FromContext(ctx).WithName("enqueueAllTenantsMapFunc") + l.Info("Enqueue all tenants") + + tenants := &synv1alpha1.TenantList{} + err := cli.List(ctx, tenants, client.InNamespace(o.GetNamespace())) + if err != nil { + l.Error(err, "Failed to list tenants") + return []reconcile.Request{} + } + + requests := make([]reconcile.Request, len(tenants.Items)) + for i, tenant := range tenants.Items { + requests[i] = reconcile.Request{ + NamespacedName: client.ObjectKey{ + Namespace: tenant.Namespace, + Name: tenant.Name, + }, + } + } + return requests + } +} diff --git a/docs/modules/ROOT/pages/references/api-reference.adoc b/docs/modules/ROOT/pages/references/api-reference.adoc index b14b102c..157397a9 100644 --- a/docs/modules/ROOT/pages/references/api-reference.adoc +++ b/docs/modules/ROOT/pages/references/api-reference.adoc @@ -22,6 +22,24 @@ TIP: A more sophisticated documentation is available under https://doc.crds.dev/ +[id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-accesstoken"] +=== AccessToken + + + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-gitrepospec[$$GitRepoSpec$$] +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-gitrepotemplate[$$GitRepoTemplate$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`secretRef`* __string__ | SecretRef references the secret the access token is stored in +|=== + + [id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-bootstraptoken"] === BootstrapToken @@ -215,6 +233,64 @@ DeployKey defines an SSH key to be used for git operations. |=== +[id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvar"] +=== EnvVar + +EnvVar represents an environment added to the CI system of the Git repository. + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-gitrepospec[$$GitRepoSpec$$] +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-gitrepotemplate[$$GitRepoTemplate$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`name`* __string__ | Name of the environment variable +| *`value`* __string__ | Value of the environment variable +| *`valueFrom`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvarsource[$$EnvVarSource$$]__ | ValueFrom is a reference to an object that contains the value of the environment variable +| *`gitlabOptions`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvargitlaboptions[$$EnvVarGitlabOptions$$]__ | GitlabOptions contains additional options for GitLab CI variables +|=== + + +[id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvargitlaboptions"] +=== EnvVarGitlabOptions + + + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvar[$$EnvVar$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`description`* __string__ | Description is a description of the CI variable. +| *`protected`* __boolean__ | Protected will expose the variable only in protected branches and tags. +| *`masked`* __boolean__ | Masked will mask the variable in the job logs. +| *`raw`* __boolean__ | Raw will prevent the variable from being expanded. +|=== + + +[id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvarsource"] +=== EnvVarSource + +EnvVarSource represents a source for the value of an EnvVar. + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvar[$$EnvVar$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`secretKeyRef`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core[$$SecretKeySelector$$]__ | Selects a key of a secret in the pod's namespace +|=== + + [id="{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-facts"] === Facts (object) @@ -289,6 +365,14 @@ Archive: will archive the external resources, if it supports that | *`creationPolicy`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-creationpolicy[$$CreationPolicy$$]__ | CreationPolicy defines how the external resources should be treated upon CR creation. Create: will only create a new external resource and will not manage already existing resources Adopt: will create a new external resource or will adopt and manage an already existing resource +| *`accessToken`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-accesstoken[$$AccessToken$$]__ | AccessToken contains configuration for storing an access token in a secret. +If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. +The token is stored under the key "token". +In the case of GitLab, this would be a Project Access Token with read-write access to the repository. +| *`ciVariables`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvar[$$EnvVar$$] array__ | CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + +The variables are not expanded like PodSpec environment variables. | *`tenantRef`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#localobjectreference-v1-core[$$LocalObjectReference$$]__ | TenantRef references the tenant this repo belongs to |=== @@ -326,6 +410,14 @@ Archive: will archive the external resources, if it supports that | *`creationPolicy`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-creationpolicy[$$CreationPolicy$$]__ | CreationPolicy defines how the external resources should be treated upon CR creation. Create: will only create a new external resource and will not manage already existing resources Adopt: will create a new external resource or will adopt and manage an already existing resource +| *`accessToken`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-accesstoken[$$AccessToken$$]__ | AccessToken contains configuration for storing an access token in a secret. +If set, the Lieutenant operator will store an access token into this secret, which can be used to access the Git repository. +The token is stored under the key "token". +In the case of GitLab, this would be a Project Access Token with read-write access to the repository. +| *`ciVariables`* __xref:{anchor_prefix}-github-com-projectsyn-lieutenant-operator-api-v1alpha1-envvar[$$EnvVar$$] array__ | CIVariables is a list of key-value pairs that will be set as CI variables in the Git repository. + + +The variables are not expanded like PodSpec environment variables. |=== diff --git a/git/gitlab/gitlab.go b/git/gitlab/gitlab.go index 05650826..0a5997c7 100644 --- a/git/gitlab/gitlab.go +++ b/git/gitlab/gitlab.go @@ -1,20 +1,25 @@ package gitlab import ( + "context" + "errors" "fmt" "net/url" + "slices" + "strconv" "strings" - - "k8s.io/utils/pointer" - - "github.com/projectsyn/lieutenant-operator/git/helpers" - "github.com/projectsyn/lieutenant-operator/git/manager" + "time" "github.com/go-logr/logr" + "github.com/xanzy/go-gitlab" + "go.uber.org/multierr" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/log" synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" - - "github.com/xanzy/go-gitlab" + "github.com/projectsyn/lieutenant-operator/git/helpers" + "github.com/projectsyn/lieutenant-operator/git/manager" ) func init() { @@ -438,13 +443,205 @@ func (g *Gitlab) compareFiles() ([]manager.CommitFile, error) { func (g *Gitlab) getCommitOptions() *gitlab.CreateCommitOptions { co := &gitlab.CreateCommitOptions{ - AuthorEmail: pointer.StringPtr("lieutenant-operator@syn.local"), - AuthorName: pointer.StringPtr("Lieutenant Operator"), - Branch: pointer.StringPtr("master"), - CommitMessage: pointer.StringPtr("Update cluster files"), + AuthorEmail: ptr.To("lieutenant-operator@syn.local"), + AuthorName: ptr.To("Lieutenant Operator"), + Branch: ptr.To("master"), + CommitMessage: ptr.To("Update cluster files"), } co.Actions = make([]*gitlab.CommitActionOptions, 0) return co } + +// EnsureProjectAccessToken ensures that the project has an access token set. +// If the token is expired or not set, a new token will be created. +// This implementation does not use the Gitlab token rotation feature. +// Using this feature would invalidate old tokens immediately, which would break pipelines. +// Also the newly created token would immediately be revoked if an old one is used. +func (g *Gitlab) EnsureProjectAccessToken(ctx context.Context, name string, opts manager.EnsureProjectAccessTokenOptions) (manager.ProjectAccessToken, error) { + at, _, err := g.client.ProjectAccessTokens.ListProjectAccessTokens(g.project.ID, &gitlab.ListProjectAccessTokensOptions{}, gitlab.WithContext(ctx)) + if err != nil { + return manager.ProjectAccessToken{}, err + } + validATs := make([]gitlab.ProjectAccessToken, 0, len(at)) + for _, token := range at { + if token == nil { + continue + } + if !token.Active { + continue + } + if token.Revoked { + continue + } + if token.ExpiresAt == nil { + continue + } + if time.Time(*token.ExpiresAt).Before(g.ops.Now().Add(-10 * 24 * time.Hour)) { + continue + } + validATs = append(validATs, *token) + } + + slices.SortFunc(validATs, func(a, b gitlab.ProjectAccessToken) int { + at := time.Time(ptr.Deref(a.ExpiresAt, gitlab.ISOTime{})) + bt := time.Time(ptr.Deref(b.ExpiresAt, gitlab.ISOTime{})) + if at.Before(bt) { + return 1 + } + if at.After(bt) { + return -1 + } + return 0 + }) + + if opts.UID == "" { + if len(validATs) > 0 { + return manager.ProjectAccessToken{ + UID: strconv.Itoa(validATs[0].ID), + ExpiresAt: time.Time(*validATs[0].ExpiresAt), + }, nil + } + } else { + for _, token := range validATs { + if strconv.Itoa(token.ID) == opts.UID { + return manager.ProjectAccessToken{ + UID: opts.UID, + ExpiresAt: time.Time(*token.ExpiresAt), + }, nil + } + } + if len(validATs) > 0 { + g.log.Info("found valid access token, but no UID match", "uid", validATs[0].ID, "given_uid", opts.UID) + } + } + + token, _, err := g.client.ProjectAccessTokens.CreateProjectAccessToken(g.project.ID, &gitlab.CreateProjectAccessTokenOptions{ + // Gitlab allows duplicated names and we can easily identify tokens by age. + // So we just reuse the name. + Name: &name, + ExpiresAt: ptr.To(gitlab.ISOTime(g.ops.Now().Add(30 * 24 * time.Hour))), + Scopes: ptr.To([]string{"write_repository"}), + AccessLevel: ptr.To(gitlab.MaintainerPermissions), + }, gitlab.WithContext(ctx)) + + if err != nil { + return manager.ProjectAccessToken{}, fmt.Errorf("error response from gitlab when creating ProjectAccessToken: %w", err) + } + + return manager.ProjectAccessToken{ + UID: strconv.Itoa(token.ID), + Token: token.Token, + ExpiresAt: time.Time(ptr.Deref(token.ExpiresAt, gitlab.ISOTime{})), + }, nil +} + +// EnsureCIVariables ensures that the given variables are set in the CI/CD pipeline. +// The managedVariables is used to identify the variables that are managed by the operator. +// Variables that are not managed by the operator will be ignored. +// Variables that are managed but not in variables will be deleted. +func (g *Gitlab) EnsureCIVariables(ctx context.Context, managedVariables []string, variables []manager.EnvVar) error { + l := log.FromContext(ctx).WithName("EnsureCIVariables") + + var errs []error + managed := sets.New(managedVariables...) + current := sets.New[string]() + for _, v := range variables { + current.Insert(v.Name) + } + + toDelete := managed.Difference(current) + for _, v := range sets.List(toDelete) { + _, err := g.client.ProjectVariables.RemoveVariable(g.project.ID, v, &gitlab.RemoveProjectVariableOptions{}, gitlab.WithContext(ctx)) + if err != nil && !errors.Is(err, gitlab.ErrNotFound) { + errs = append(errs, fmt.Errorf("error removing variable %s: %w", v, err)) + } + } + + remote, _, err := g.client.ProjectVariables.ListVariables(g.project.ID, &gitlab.ListProjectVariablesOptions{}, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("error listing variables: %w", err) + } + remoteByName := make(map[string]gitlab.ProjectVariable, len(remote)) + for _, v := range remote { + if v == nil { + continue + } + remoteByName[v.Key] = *v + } + + for _, v := range variables { + if !managed.Has(v.Name) { + continue + } + + remote, ok := remoteByName[v.Name] + var changed bool + if ok { + changed = varNeedsUpdate(remote, v) + } else { + changed = true + } + if !changed { + continue + } + + l.Info("updating changed variable", "name", v.Name) + err := g.updateOrCreateCIVariable(ctx, v) + if err != nil { + errs = append(errs, err) + } + } + + return multierr.Combine(errs...) +} + +// varNeedsUpdate returns true if the remote variable needs to be updated. +// Does not check the Key, as the key is not allowed to change. +func varNeedsUpdate(remote gitlab.ProjectVariable, local manager.EnvVar) bool { + if remote.Value != local.Value { + return true + } + if local.GitlabOptions.Description != nil && remote.Description != *local.GitlabOptions.Description { + return true + } + if local.GitlabOptions.Protected != nil && remote.Protected != *local.GitlabOptions.Protected { + return true + } + if local.GitlabOptions.Masked != nil && remote.Masked != *local.GitlabOptions.Masked { + return true + } + if local.GitlabOptions.Raw != nil && remote.Raw != *local.GitlabOptions.Raw { + return true + } + return false +} + +// updateOrCreateCIVariable updates or creates a CI variable in the Gitlab project. +// It tries to update the variable first and creates it if it does not exist. +func (g *Gitlab) updateOrCreateCIVariable(ctx context.Context, v manager.EnvVar) error { + _, _, err := g.client.ProjectVariables.UpdateVariable(g.project.ID, v.Name, &gitlab.UpdateProjectVariableOptions{ + Value: &v.Value, + Protected: v.GitlabOptions.Protected, + Masked: v.GitlabOptions.Masked, + Raw: v.GitlabOptions.Raw, + }, gitlab.WithContext(ctx)) + if err == nil { + return nil + } + if !errors.Is(err, gitlab.ErrNotFound) { + return fmt.Errorf("error updating variable %s: %w", v.Name, err) + } + _, _, err = g.client.ProjectVariables.CreateVariable(g.project.ID, &gitlab.CreateProjectVariableOptions{ + Key: &v.Name, + Value: &v.Value, + Protected: v.GitlabOptions.Protected, + Masked: v.GitlabOptions.Masked, + Raw: v.GitlabOptions.Raw, + }, gitlab.WithContext(ctx)) + if err != nil { + return fmt.Errorf("error creating variable %s: %w", v.Name, err) + } + return nil +} diff --git a/git/gitlab/gitlab_test.go b/git/gitlab/gitlab_test.go index d09dfe94..cfab6be0 100644 --- a/git/gitlab/gitlab_test.go +++ b/git/gitlab/gitlab_test.go @@ -2,20 +2,29 @@ package gitlab import ( "bytes" + "context" + _ "embed" "encoding/json" "fmt" "io" "net/http" "net/http/httptest" "net/url" + "slices" "strconv" + "strings" + "sync" "testing" + "time" "github.com/go-logr/zapr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/xanzy/go-gitlab" + "go.uber.org/atomic" "go.uber.org/zap" + "golang.org/x/exp/maps" + "k8s.io/utils/ptr" "github.com/projectsyn/lieutenant-operator/git/manager" "github.com/projectsyn/lieutenant-operator/testutils" @@ -249,6 +258,15 @@ func TestGitlab_Delete(t *testing.T) { } } +//go:embed testdata/testGetUpdateServer/deploy_keys.json +var keysResponse []byte + +//go:embed testdata/testGetUpdateServer/projects_path_response.json +var projectsPathResponse []byte + +//go:embed testdata/testGetUpdateServer/projects_id_response.json +var projectsIDResponse []byte + //goland:noinspection HttpUrlsUsage func testGetUpdateServer(t *testing.T, fail bool) *httptest.Server { mux := http.NewServeMux() @@ -260,13 +278,13 @@ func testGetUpdateServer(t *testing.T, fail bool) *httptest.Server { respH = http.StatusInternalServerError } res.WriteHeader(respH) - _, _ = res.Write([]byte(`[{"id":1,"title":"Public key","key":"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=","created_at":"2013-10-02T10:12:29Z","can_push":false},{"id":3,"title":"Another Public key","key":"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=","created_at":"2013-10-02T11:12:29Z","can_push":false}]`)) + _, _ = res.Write(keysResponse) }) mux.HandleFunc("/api/v4/projects/updated%2Frepo", func(res http.ResponseWriter, req *http.Request) { respH := http.StatusOK res.WriteHeader(respH) - _, _ = res.Write([]byte(`{"id":3,"description":"oldDesc","default_branch":"master","visibility":"private","ssh_url_to_repo":"git@example.com:luzifern/luzifern-project-site.git","http_url_to_repo":"http://example.com/diaspora/diaspora-project-site.git","web_url":"http://example.com/diaspora/diaspora-project-site","readme_url":"http://example.com/diaspora/diaspora-project-site/blob/master/README.md","tag_list":["example","disapora project"],"owner":{"id":3,"name":"Diaspora","created_at":"2013-09-30T13:46:02Z"},"name":"repo","name_with_namespace":"group1 / Diaspora Project Site","path":"diaspora-project-site","path_with_namespace":"group1/diaspora-project-site","issues_enabled":true,"open_issues_count":1,"merge_requests_enabled":true,"jobs_enabled":true,"wiki_enabled":true,"snippets_enabled":false,"resolve_outdated_diff_discussions":false,"container_registry_enabled":false,"container_expiration_policy":{"cadence":"7d","enabled":false,"keep_n":null,"older_than":null,"name_regex":null,"next_run_at":"2020-01-07T21:42:58.658Z"},"created_at":"2013-09-30T13:46:02Z","last_activity_at":"2013-09-30T13:46:02Z","creator_id":2,"namespace":{"id":2,"name":"group1","path":"group1","kind":"group","full_path":"group1","parent_id":null,"members_count_with_descendants":2},"import_status":"none","import_error":null,"permissions":{"project_access":{"access_level":10,"notification_level":3},"group_access":{"access_level":50,"notification_level":3}},"archived":false,"avatar_url":"http://example.com/uploads/project/avatar/3/uploads/avatar.png","license_url":"http://example.com/diaspora/diaspora-client/blob/master/LICENSE","license":{"key":"lgpl-3.0","name":"GNU Lesser General Public License v3.0","nickname":"GNU LGPLv3","html_url":"http://choosealicense.com/licenses/lgpl-3.0/","source_url":"http://www.gnu.org/licenses/lgpl-3.0.txt"},"shared_runners_enabled":true,"forks_count":0,"star_count":0,"runners_token":"b8bc4a7a29eb76ea83cf79e4908c2b","ci_default_git_depth":50,"public_jobs":true,"shared_with_groups":[{"group_id":4,"group_name":"Twitter","group_full_path":"twitter","group_access_level":30},{"group_id":3,"group_name":"Gitlab Org","group_full_path":"gitlab-org","group_access_level":10}],"repository_storage":"default","only_allow_merge_if_pipeline_succeeds":false,"only_allow_merge_if_all_discussions_are_resolved":false,"remove_source_branch_after_merge":false,"printing_merge_requests_link_enabled":true,"request_access_enabled":false,"merge_method":"merge","auto_devops_enabled":true,"auto_devops_deploy_strategy":"continuous","approvals_before_merge":0,"mirror":false,"mirror_user_id":45,"mirror_trigger_builds":false,"only_mirror_protected_branches":false,"mirror_overwrites_diverged_branches":false,"external_authorization_classification_label":null,"packages_enabled":true,"service_desk_enabled":false,"service_desk_address":null,"autoclose_referenced_issues":true,"suggestion_commit_message":null,"statistics":{"commit_count":37,"storage_size":1038090,"repository_size":1038090,"wiki_size":0,"lfs_objects_size":0,"job_artifacts_size":0,"packages_size":0},"_links":{"self":"http://example.com/api/v4/projects","issues":"http://example.com/api/v4/projects/1/issues","merge_requests":"http://example.com/api/v4/projects/1/merge_requests","repo_branches":"http://example.com/api/v4/projects/1/repository_branches","labels":"http://example.com/api/v4/projects/1/labels","events":"http://example.com/api/v4/projects/1/events","members":"http://example.com/api/v4/projects/1/members"}}`)) + _, _ = res.Write([]byte(projectsPathResponse)) }) mux.HandleFunc("/api/v4/projects/3", func(res http.ResponseWriter, req *http.Request) { @@ -279,7 +297,26 @@ func testGetUpdateServer(t *testing.T, fail bool) *httptest.Server { response = http.StatusInternalServerError } res.WriteHeader(response) - _, _ = res.Write([]byte(`{"id":3,"description":"` + *editProjectOptions.Description + `","default_branch":"master","visibility":"private","ssh_url_to_repo":"git@example.com:luzifern/luzifern-project-site.git","http_url_to_repo":"http://example.com/diaspora/diaspora-project-site.git","web_url":"http://example.com/diaspora/diaspora-project-site","readme_url":"http://example.com/diaspora/diaspora-project-site/blob/master/README.md","tag_list":["example","disapora project"],"owner":{"id":3,"name":"Diaspora","created_at":"2013-09-30T13:46:02Z"},"name":"repo","name_with_namespace":"group1 / Diaspora Project Site","path":"diaspora-project-site","path_with_namespace":"group1/diaspora-project-site","issues_enabled":true,"open_issues_count":1,"merge_requests_enabled":true,"jobs_enabled":true,"wiki_enabled":true,"snippets_enabled":false,"resolve_outdated_diff_discussions":false,"container_registry_enabled":false,"container_expiration_policy":{"cadence":"7d","enabled":false,"keep_n":null,"older_than":null,"name_regex":null,"next_run_at":"2020-01-07T21:42:58.658Z"},"created_at":"2013-09-30T13:46:02Z","last_activity_at":"2013-09-30T13:46:02Z","creator_id":2,"namespace":{"id":2,"name":"group1","path":"group1","kind":"group","full_path":"group1","parent_id":null,"members_count_with_descendants":2},"import_status":"none","import_error":null,"permissions":{"project_access":{"access_level":10,"notification_level":3},"group_access":{"access_level":50,"notification_level":3}},"archived":false,"avatar_url":"http://example.com/uploads/project/avatar/3/uploads/avatar.png","license_url":"http://example.com/diaspora/diaspora-client/blob/master/LICENSE","license":{"key":"lgpl-3.0","name":"GNU Lesser General Public License v3.0","nickname":"GNU LGPLv3","html_url":"http://choosealicense.com/licenses/lgpl-3.0/","source_url":"http://www.gnu.org/licenses/lgpl-3.0.txt"},"shared_runners_enabled":true,"forks_count":0,"star_count":0,"runners_token":"b8bc4a7a29eb76ea83cf79e4908c2b","ci_default_git_depth":50,"public_jobs":true,"shared_with_groups":[{"group_id":4,"group_name":"Twitter","group_full_path":"twitter","group_access_level":30},{"group_id":3,"group_name":"Gitlab Org","group_full_path":"gitlab-org","group_access_level":10}],"repository_storage":"default","only_allow_merge_if_pipeline_succeeds":false,"only_allow_merge_if_all_discussions_are_resolved":false,"remove_source_branch_after_merge":false,"printing_merge_requests_link_enabled":true,"request_access_enabled":false,"merge_method":"merge","auto_devops_enabled":true,"auto_devops_deploy_strategy":"continuous","approvals_before_merge":0,"mirror":false,"mirror_user_id":45,"mirror_trigger_builds":false,"only_mirror_protected_branches":false,"mirror_overwrites_diverged_branches":false,"external_authorization_classification_label":null,"packages_enabled":true,"service_desk_enabled":false,"service_desk_address":null,"autoclose_referenced_issues":true,"suggestion_commit_message":null,"statistics":{"commit_count":37,"storage_size":1038090,"repository_size":1038090,"wiki_size":0,"lfs_objects_size":0,"job_artifacts_size":0,"packages_size":0},"_links":{"self":"http://example.com/api/v4/projects","issues":"http://example.com/api/v4/projects/1/issues","merge_requests":"http://example.com/api/v4/projects/1/merge_requests","repo_branches":"http://example.com/api/v4/projects/1/repository_branches","labels":"http://example.com/api/v4/projects/1/labels","events":"http://example.com/api/v4/projects/1/events","members":"http://example.com/api/v4/projects/1/members"}}`)) + + var project gitlab.Project + if err := json.Unmarshal(projectsIDResponse, &project); err != nil { + res.WriteHeader(http.StatusInternalServerError) + t.Logf("unmarshal failed: %v", err) + _, _ = res.Write([]byte(`{"error":"test server error: unmarshal failed"}`)) + return + } + + project.Description = *editProjectOptions.Description + + projectBytes, err := json.Marshal(project) + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + t.Logf("marshal failed: %v", err) + _, _ = res.Write([]byte(`{"error":"test server error: marshal failed"}`)) + return + } + + _, _ = res.Write([]byte(projectBytes)) }) deleteOk := func(res http.ResponseWriter, req *http.Request) { @@ -601,3 +638,332 @@ func TestGitlab_FullURL(t *testing.T) { assert.Equal(t, expectedFullURL, g.FullURL().String()) assert.Equal(t, expectedFullURL, g.FullURL().String()) } + +type mockClock struct { + now time.Time +} + +func (m *mockClock) Now() time.Time { + return m.now +} + +func (m *mockClock) Advance(d time.Duration) { + m.now = m.now.Add(d) +} + +func TestGitlab_EnsureProjectAccessToken(t *testing.T) { + clock := &mockClock{now: time.Now()} + + serv := testProjectAccessTokenServer(t, clock.Now) + defer serv.Close() + + url, err := url.Parse(serv.URL) + require.NoError(t, err) + + g := &Gitlab{ + project: &gitlab.Project{ + ID: 3, + }, + ops: manager.RepoOptions{ + URL: url, + Clock: clock, + }, + } + + require.NoError(t, g.Connect()) + + pat, err := g.EnsureProjectAccessToken(context.Background(), "test", manager.EnsureProjectAccessTokenOptions{}) + require.NoError(t, err) + assert.Equal(t, "token101", pat.Token) + + for _, uid := range []string{"", pat.UID} { + opts := manager.EnsureProjectAccessTokenOptions{UID: uid} + samepat, err := g.EnsureProjectAccessToken(context.Background(), "test", opts) + require.NoError(t, err) + assert.Equal(t, pat.UID, samepat.UID, "Should reuse the same token", "opts", opts) + assert.Equal(t, pat.ExpiresAt, samepat.ExpiresAt) + } + + newPat, err := g.EnsureProjectAccessToken(context.Background(), "test", manager.EnsureProjectAccessTokenOptions{UID: "other id"}) + require.NoError(t, err) + assert.NotEqual(t, pat.UID, newPat.UID, "Should return new token if UID does not match") + + // Access token expiry are floored to the nearest day + // Check that newest token is returned + clock.Advance(24 * time.Hour) + newerPat, err := g.EnsureProjectAccessToken(context.Background(), "test", manager.EnsureProjectAccessTokenOptions{UID: "other id"}) + require.NoError(t, err) + assert.NotEqual(t, newPat.UID, newerPat.UID, "Should return new token if UID does not match") + newerPat2, err := g.EnsureProjectAccessToken(context.Background(), "test", manager.EnsureProjectAccessTokenOptions{}) + require.NoError(t, err) + assert.Equal(t, newerPat.UID, newerPat2.UID, "Should return the newest created token") + + clock.Advance(time.Hour * 24 * 90) + + renewedPat, err := g.EnsureProjectAccessToken(context.Background(), "test", manager.EnsureProjectAccessTokenOptions{UID: pat.UID}) + require.NoError(t, err) + assert.NotEmpty(t, renewedPat.Token, "Should return new token if old token is expired") + assert.NotEqual(t, pat.UID, renewedPat.UID, "Should return new token if old token is expired") +} + +func TestGitlab_EnsureCIVariables(t *testing.T) { + clock := &mockClock{now: time.Now()} + + serv := newTestProjectProjectVariablesServer(t, clock.Now) + defer serv.Close() + + url, err := url.Parse(serv.URL) + require.NoError(t, err) + + g := &Gitlab{ + project: &gitlab.Project{ + ID: 3, + }, + ops: manager.RepoOptions{ + URL: url, + }, + } + + require.NoError(t, g.Connect()) + + vars := []manager.EnvVar{ + { + Name: "KEY1", + Value: "value1", + }, + { + Name: "KEY2", + Value: "value2", + GitlabOptions: manager.EnvVarGitlabOptions{ + Protected: ptr.To(true), + }, + }, + } + + // Create variables + require.NoError(t, g.EnsureCIVariables(context.Background(), []string{"KEY1", "KEY2"}, vars)) + cvs, _, err := g.client.ProjectVariables.ListVariables(g.project.ID, &gitlab.ListProjectVariablesOptions{}) + require.NoError(t, err) + require.Len(t, cvs, 2) + assert.Equal(t, "KEY1", cvs[0].Key) + assert.Equal(t, "value1", cvs[0].Value) + assert.False(t, cvs[0].Protected) + assert.Equal(t, "KEY2", cvs[1].Key) + assert.Equal(t, "value2", cvs[1].Value) + assert.True(t, cvs[1].Protected) + + // no changes should be write noops + prevCalls := serv.updateCount.Load() + require.NoError(t, g.EnsureCIVariables(context.Background(), []string{"KEY1", "KEY2"}, vars)) + require.Equal(t, prevCalls, serv.updateCount.Load(), "no changes should be write noops") + + // Update variable value + vars[0].Value = "value1.1" + vars[1].Value = "value2.1" + require.NoError(t, g.EnsureCIVariables(context.Background(), []string{"KEY1"}, vars)) + cvs, _, err = g.client.ProjectVariables.ListVariables(g.project.ID, &gitlab.ListProjectVariablesOptions{}) + require.NoError(t, err) + require.Len(t, cvs, 2) + assert.Equal(t, "value1.1", cvs[0].Value) + assert.Equal(t, "value2", cvs[1].Value, "should not update unmanaged variable") + + // Update variable advanced options + vars[0].GitlabOptions.Protected = ptr.To(true) + vars[1].GitlabOptions.Protected = ptr.To(false) + require.NoError(t, g.EnsureCIVariables(context.Background(), []string{"KEY1", "KEY2"}, vars)) + cvs, _, err = g.client.ProjectVariables.ListVariables(g.project.ID, &gitlab.ListProjectVariablesOptions{}) + require.NoError(t, err) + require.Len(t, cvs, 2) + assert.True(t, cvs[0].Protected) + assert.False(t, cvs[1].Protected) + + // Delete variable + require.NoError(t, g.EnsureCIVariables(context.Background(), []string{"KEY1"}, nil)) + cvs, _, err = g.client.ProjectVariables.ListVariables(g.project.ID, &gitlab.ListProjectVariablesOptions{}) + require.NoError(t, err) + require.Len(t, cvs, 1, "should delete managed variable") + assert.Equal(t, "KEY2", cvs[0].Key, "should not delete unmanaged variable") +} + +func testProjectAccessTokenServer(t *testing.T, clock func() time.Time) *httptest.Server { + mux := http.NewServeMux() + + var patsMux sync.Mutex + var pats []gitlab.ProjectAccessToken + + mux.HandleFunc("GET /api/v4/projects/3/access_tokens", func(res http.ResponseWriter, req *http.Request) { + patsMux.Lock() + defer patsMux.Unlock() + _ = json.NewEncoder(res).Encode(pats) + }) + + mux.HandleFunc("POST /api/v4/projects/3/access_tokens", func(res http.ResponseWriter, req *http.Request) { + var createPAT gitlab.CreateProjectAccessTokenOptions + if err := json.NewDecoder(req.Body).Decode(&createPAT); err != nil { + res.WriteHeader(http.StatusInternalServerError) + t.Logf("unmarshal failed: %v", err) + _, _ = res.Write([]byte(`{"error":"unmarshal failed"}`)) + return + } + + patsMux.Lock() + id := 100 + len(pats) + 1 + nPat := gitlab.ProjectAccessToken{ + ID: id, + UserID: 123, + Name: *createPAT.Name, + Scopes: *createPAT.Scopes, + CreatedAt: ptr.To(clock()), + ExpiresAt: createPAT.ExpiresAt, + Active: true, + Revoked: false, + Token: "token" + strconv.Itoa(id), + AccessLevel: *createPAT.AccessLevel, + } + pats = append(pats, nPat) + patsMux.Unlock() + _ = json.NewEncoder(res).Encode(nPat) + }) + + mux.HandleFunc("/", testutils.LogNotFoundHandler(t)) + + return httptest.NewServer(mux) +} + +type testProjectProjectVariablesServer struct { + *httptest.Server + + varsMux sync.Mutex + vars map[string]gitlab.ProjectVariable + + createCount atomic.Int32 + updateCount atomic.Int32 + deleteCount atomic.Int32 +} + +func newTestProjectProjectVariablesServer(t *testing.T, clock func() time.Time) *testProjectProjectVariablesServer { + mux := http.NewServeMux() + + s := &testProjectProjectVariablesServer{ + vars: make(map[string]gitlab.ProjectVariable), + } + + mux.HandleFunc("GET /api/v4/projects/3/variables", func(res http.ResponseWriter, req *http.Request) { + s.varsMux.Lock() + defer s.varsMux.Unlock() + vs := maps.Values(s.vars) + slices.SortFunc(vs, func(a, b gitlab.ProjectVariable) int { + return strings.Compare(a.Key, b.Key) + }) + _ = json.NewEncoder(res).Encode(vs) + }) + + mux.HandleFunc("POST /api/v4/projects/3/variables", func(res http.ResponseWriter, req *http.Request) { + s.createCount.Inc() + + var createVar gitlab.CreateProjectVariableOptions + if err := json.NewDecoder(req.Body).Decode(&createVar); err != nil { + res.WriteHeader(http.StatusInternalServerError) + t.Logf("unmarshal failed: %v", err) + _, _ = res.Write([]byte(`{"error":"unmarshal failed"}`)) + return + } + + if createVar.Key == nil { + res.WriteHeader(http.StatusBadRequest) + _, _ = res.Write([]byte(`{"error":"key is required"}`)) + return + } + key := *createVar.Key + + s.varsMux.Lock() + if _, ok := s.vars[key]; ok { + res.WriteHeader(http.StatusBadRequest) + _, _ = res.Write([]byte(`{"error":"variable already exists"}`)) + s.varsMux.Unlock() + return + } + defer s.varsMux.Unlock() + nVar := gitlab.ProjectVariable{ + Key: key, + Value: ptr.Deref(createVar.Value, ""), + VariableType: ptr.Deref(createVar.VariableType, gitlab.EnvVariableType), + Protected: ptr.Deref(createVar.Protected, false), + Masked: ptr.Deref(createVar.Masked, false), + Raw: ptr.Deref(createVar.Raw, false), + EnvironmentScope: ptr.Deref(createVar.EnvironmentScope, "*"), + Description: ptr.Deref(createVar.Description, ""), + } + s.vars[key] = nVar + _ = json.NewEncoder(res).Encode(nVar) + }) + + mux.HandleFunc("PUT /api/v4/projects/3/variables/{key}", func(res http.ResponseWriter, req *http.Request) { + s.updateCount.Inc() + + var createVar gitlab.UpdateProjectVariableOptions + if err := json.NewDecoder(req.Body).Decode(&createVar); err != nil { + res.WriteHeader(http.StatusInternalServerError) + t.Logf("unmarshal failed: %v", err) + _, _ = res.Write([]byte(`{"error":"unmarshal failed"}`)) + return + } + + key := req.PathValue("key") + if key == "" { + res.WriteHeader(http.StatusBadRequest) + _, _ = res.Write([]byte(`{"error":"key is required"}`)) + return + } + + s.varsMux.Lock() + oVar, ok := s.vars[key] + if !ok { + res.WriteHeader(http.StatusNotFound) + _, _ = res.Write([]byte(`{"error":"404 not found"}`)) + s.varsMux.Unlock() + return + } + defer s.varsMux.Unlock() + nVar := gitlab.ProjectVariable{ + Key: key, + Value: ptr.Deref(createVar.Value, oVar.Value), + VariableType: ptr.Deref(createVar.VariableType, oVar.VariableType), + Protected: ptr.Deref(createVar.Protected, oVar.Protected), + Masked: ptr.Deref(createVar.Masked, oVar.Masked), + Raw: ptr.Deref(createVar.Raw, oVar.Raw), + EnvironmentScope: ptr.Deref(createVar.EnvironmentScope, oVar.EnvironmentScope), + Description: ptr.Deref(createVar.Description, oVar.Description), + } + s.vars[key] = nVar + _ = json.NewEncoder(res).Encode(nVar) + }) + + mux.HandleFunc("DELETE /api/v4/projects/3/variables/{key}", func(res http.ResponseWriter, req *http.Request) { + s.deleteCount.Inc() + + key := req.PathValue("key") + if key == "" { + res.WriteHeader(http.StatusBadRequest) + _, _ = res.Write([]byte(`{"error":"key is required"}`)) + return + } + + s.varsMux.Lock() + oVar, ok := s.vars[key] + if !ok { + res.WriteHeader(http.StatusNotFound) + _, _ = res.Write([]byte(`{"error":"404 not found"}`)) + s.varsMux.Unlock() + return + } + defer s.varsMux.Unlock() + delete(s.vars, key) + _ = json.NewEncoder(res).Encode(oVar) + }) + + mux.HandleFunc("/", testutils.LogNotFoundHandler(t)) + + s.Server = httptest.NewServer(mux) + return s +} diff --git a/git/gitlab/testdata/testGetUpdateServer/deploy_keys.json b/git/gitlab/testdata/testGetUpdateServer/deploy_keys.json new file mode 100644 index 00000000..a7c85231 --- /dev/null +++ b/git/gitlab/testdata/testGetUpdateServer/deploy_keys.json @@ -0,0 +1,16 @@ +[ + { + "id": 1, + "title": "Public key", + "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", + "created_at": "2013-10-02T10:12:29Z", + "can_push": false + }, + { + "id": 3, + "title": "Another Public key", + "key": "ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAIEAiPWx6WM4lhHNedGfBpPJNPpZ7yKu+dnn1SJejgt4596k6YjzGGphH2TUxwKzxcKDKKezwkpfnxPkSMkuEspGRt/aZZ9wa++Oi7Qkr8prgHc4soW6NUlfDzpvZK2H5E7eQaSeP3SAwGmQKUFHCddNaP0L+hM7zhFNzjFvpaMgJw0=", + "created_at": "2013-10-02T11:12:29Z", + "can_push": false + } +] diff --git a/git/gitlab/testdata/testGetUpdateServer/projects_id_response.json b/git/gitlab/testdata/testGetUpdateServer/projects_id_response.json new file mode 100644 index 00000000..2c41b44c --- /dev/null +++ b/git/gitlab/testdata/testGetUpdateServer/projects_id_response.json @@ -0,0 +1,123 @@ +{ + "id": 3, + "description": "XXX", + "default_branch": "master", + "visibility": "private", + "ssh_url_to_repo": "git@example.com:luzifern/luzifern-project-site.git", + "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", + "web_url": "http://example.com/diaspora/diaspora-project-site", + "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", + "tag_list": ["example", "disapora project"], + "owner": { + "id": 3, + "name": "Diaspora", + "created_at": "2013-09-30T13:46:02Z" + }, + "name": "repo", + "name_with_namespace": "group1 / Diaspora Project Site", + "path": "diaspora-project-site", + "path_with_namespace": "group1/diaspora-project-site", + "issues_enabled": true, + "open_issues_count": 1, + "merge_requests_enabled": true, + "jobs_enabled": true, + "wiki_enabled": true, + "snippets_enabled": false, + "resolve_outdated_diff_discussions": false, + "container_registry_enabled": false, + "container_expiration_policy": { + "cadence": "7d", + "enabled": false, + "keep_n": null, + "older_than": null, + "name_regex": null, + "next_run_at": "2020-01-07T21:42:58.658Z" + }, + "created_at": "2013-09-30T13:46:02Z", + "last_activity_at": "2013-09-30T13:46:02Z", + "creator_id": 2, + "namespace": { + "id": 2, + "name": "group1", + "path": "group1", + "kind": "group", + "full_path": "group1", + "parent_id": null, + "members_count_with_descendants": 2 + }, + "import_status": "none", + "import_error": null, + "permissions": { + "project_access": { "access_level": 10, "notification_level": 3 }, + "group_access": { "access_level": 50, "notification_level": 3 } + }, + "archived": false, + "avatar_url": "http://example.com/uploads/project/avatar/3/uploads/avatar.png", + "license_url": "http://example.com/diaspora/diaspora-client/blob/master/LICENSE", + "license": { + "key": "lgpl-3.0", + "name": "GNU Lesser General Public License v3.0", + "nickname": "GNU LGPLv3", + "html_url": "http://choosealicense.com/licenses/lgpl-3.0/", + "source_url": "http://www.gnu.org/licenses/lgpl-3.0.txt" + }, + "shared_runners_enabled": true, + "forks_count": 0, + "star_count": 0, + "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", + "ci_default_git_depth": 50, + "public_jobs": true, + "shared_with_groups": [ + { + "group_id": 4, + "group_name": "Twitter", + "group_full_path": "twitter", + "group_access_level": 30 + }, + { + "group_id": 3, + "group_name": "Gitlab Org", + "group_full_path": "gitlab-org", + "group_access_level": 10 + } + ], + "repository_storage": "default", + "only_allow_merge_if_pipeline_succeeds": false, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": false, + "printing_merge_requests_link_enabled": true, + "request_access_enabled": false, + "merge_method": "merge", + "auto_devops_enabled": true, + "auto_devops_deploy_strategy": "continuous", + "approvals_before_merge": 0, + "mirror": false, + "mirror_user_id": 45, + "mirror_trigger_builds": false, + "only_mirror_protected_branches": false, + "mirror_overwrites_diverged_branches": false, + "external_authorization_classification_label": null, + "packages_enabled": true, + "service_desk_enabled": false, + "service_desk_address": null, + "autoclose_referenced_issues": true, + "suggestion_commit_message": null, + "statistics": { + "commit_count": 37, + "storage_size": 1038090, + "repository_size": 1038090, + "wiki_size": 0, + "lfs_objects_size": 0, + "job_artifacts_size": 0, + "packages_size": 0 + }, + "_links": { + "self": "http://example.com/api/v4/projects", + "issues": "http://example.com/api/v4/projects/1/issues", + "merge_requests": "http://example.com/api/v4/projects/1/merge_requests", + "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", + "labels": "http://example.com/api/v4/projects/1/labels", + "events": "http://example.com/api/v4/projects/1/events", + "members": "http://example.com/api/v4/projects/1/members" + } +} diff --git a/git/gitlab/testdata/testGetUpdateServer/projects_path_response.json b/git/gitlab/testdata/testGetUpdateServer/projects_path_response.json new file mode 100644 index 00000000..681d2f8a --- /dev/null +++ b/git/gitlab/testdata/testGetUpdateServer/projects_path_response.json @@ -0,0 +1,123 @@ +{ + "id": 3, + "description": "oldDesc", + "default_branch": "master", + "visibility": "private", + "ssh_url_to_repo": "git@example.com:luzifern/luzifern-project-site.git", + "http_url_to_repo": "http://example.com/diaspora/diaspora-project-site.git", + "web_url": "http://example.com/diaspora/diaspora-project-site", + "readme_url": "http://example.com/diaspora/diaspora-project-site/blob/master/README.md", + "tag_list": ["example", "disapora project"], + "owner": { + "id": 3, + "name": "Diaspora", + "created_at": "2013-09-30T13:46:02Z" + }, + "name": "repo", + "name_with_namespace": "group1 / Diaspora Project Site", + "path": "diaspora-project-site", + "path_with_namespace": "group1/diaspora-project-site", + "issues_enabled": true, + "open_issues_count": 1, + "merge_requests_enabled": true, + "jobs_enabled": true, + "wiki_enabled": true, + "snippets_enabled": false, + "resolve_outdated_diff_discussions": false, + "container_registry_enabled": false, + "container_expiration_policy": { + "cadence": "7d", + "enabled": false, + "keep_n": null, + "older_than": null, + "name_regex": null, + "next_run_at": "2020-01-07T21:42:58.658Z" + }, + "created_at": "2013-09-30T13:46:02Z", + "last_activity_at": "2013-09-30T13:46:02Z", + "creator_id": 2, + "namespace": { + "id": 2, + "name": "group1", + "path": "group1", + "kind": "group", + "full_path": "group1", + "parent_id": null, + "members_count_with_descendants": 2 + }, + "import_status": "none", + "import_error": null, + "permissions": { + "project_access": { "access_level": 10, "notification_level": 3 }, + "group_access": { "access_level": 50, "notification_level": 3 } + }, + "archived": false, + "avatar_url": "http://example.com/uploads/project/avatar/3/uploads/avatar.png", + "license_url": "http://example.com/diaspora/diaspora-client/blob/master/LICENSE", + "license": { + "key": "lgpl-3.0", + "name": "GNU Lesser General Public License v3.0", + "nickname": "GNU LGPLv3", + "html_url": "http://choosealicense.com/licenses/lgpl-3.0/", + "source_url": "http://www.gnu.org/licenses/lgpl-3.0.txt" + }, + "shared_runners_enabled": true, + "forks_count": 0, + "star_count": 0, + "runners_token": "b8bc4a7a29eb76ea83cf79e4908c2b", + "ci_default_git_depth": 50, + "public_jobs": true, + "shared_with_groups": [ + { + "group_id": 4, + "group_name": "Twitter", + "group_full_path": "twitter", + "group_access_level": 30 + }, + { + "group_id": 3, + "group_name": "Gitlab Org", + "group_full_path": "gitlab-org", + "group_access_level": 10 + } + ], + "repository_storage": "default", + "only_allow_merge_if_pipeline_succeeds": false, + "only_allow_merge_if_all_discussions_are_resolved": false, + "remove_source_branch_after_merge": false, + "printing_merge_requests_link_enabled": true, + "request_access_enabled": false, + "merge_method": "merge", + "auto_devops_enabled": true, + "auto_devops_deploy_strategy": "continuous", + "approvals_before_merge": 0, + "mirror": false, + "mirror_user_id": 45, + "mirror_trigger_builds": false, + "only_mirror_protected_branches": false, + "mirror_overwrites_diverged_branches": false, + "external_authorization_classification_label": null, + "packages_enabled": true, + "service_desk_enabled": false, + "service_desk_address": null, + "autoclose_referenced_issues": true, + "suggestion_commit_message": null, + "statistics": { + "commit_count": 37, + "storage_size": 1038090, + "repository_size": 1038090, + "wiki_size": 0, + "lfs_objects_size": 0, + "job_artifacts_size": 0, + "packages_size": 0 + }, + "_links": { + "self": "http://example.com/api/v4/projects", + "issues": "http://example.com/api/v4/projects/1/issues", + "merge_requests": "http://example.com/api/v4/projects/1/merge_requests", + "repo_branches": "http://example.com/api/v4/projects/1/repository_branches", + "labels": "http://example.com/api/v4/projects/1/labels", + "events": "http://example.com/api/v4/projects/1/events", + "members": "http://example.com/api/v4/projects/1/members" + } +} diff --git a/git/manager/manager.go b/git/manager/manager.go index 4cb6a2d4..21e496d5 100644 --- a/git/manager/manager.go +++ b/git/manager/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "time" synv1alpha1 "github.com/projectsyn/lieutenant-operator/api/v1alpha1" "k8s.io/apimachinery/pkg/types" @@ -69,6 +70,20 @@ type RepoOptions struct { DisplayName string TemplateFiles map[string]string DeletionPolicy synv1alpha1.DeletionPolicy + + // Clock is used to get the current time. It is used to mock the time in tests. + // If not set, time.Now() will be used. + Clock interface { + Now() time.Time + } +} + +// Now returns the current time. If the clock is not set, time.Now() will be used. +func (r RepoOptions) Now() time.Time { + if r.Clock == nil { + return time.Now() + } + return r.Clock.Now() } // Credentials holds the authentication information for the API. Most of the times this @@ -97,6 +112,50 @@ type Repo interface { // files that contain exactly the deletion magic string should be removed // when calling this function. TODO: will be replaced with something better in the future. CommitTemplateFiles() error + // EnsureProjectAccessToken will ensure that the project access token is set in the repository. + // If the token is expired or not set, a new token will be created. + // Depending on the implementation the token name might be used as a prefix. + EnsureProjectAccessToken(ctx context.Context, name string, opts EnsureProjectAccessTokenOptions) (ProjectAccessToken, error) + // EnsureCIVariables will ensure that the given variables are set in the CI/CD pipeline. + // The managedVariables is used to identify the variables that are managed by the operator. + // Variables that are not managed by the operator will be ignored. + // Variables that are managed but not in variables will be deleted. + EnsureCIVariables(ctx context.Context, managedVariables []string, variables []EnvVar) error +} + +// EnvVar represents a CI/CD environment variable. +// It can have manager specific options. +// The manager specific options are ignored if the manager does not support them. +type EnvVar struct { + Name string + Value string + + GitlabOptions EnvVarGitlabOptions +} + +type EnvVarGitlabOptions struct { + Description *string + Protected *bool + Masked *bool + Raw *bool +} + +type EnsureProjectAccessTokenOptions struct { + // UID is a unique identifier for the token. + // If set, the given UID will be compared with the UID of the existing token. + // The token will be force updated if the UIDs do not match. + UID string +} + +type ProjectAccessToken struct { + UID string + Token string + ExpiresAt time.Time +} + +// Updated returns true if the token was updated +func (p ProjectAccessToken) Updated() bool { + return p.Token != "" } // Implementation is a set of functions needed to get the right git implementation diff --git a/go.mod b/go.mod index 166ecf5e..297b6003 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,10 @@ require ( github.com/ryankurte/go-structparse v1.2.0 github.com/stretchr/testify v1.9.0 github.com/xanzy/go-gitlab v0.106.0 + go.uber.org/atomic v1.11.0 + go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 k8s.io/api v0.30.1 k8s.io/apimachinery v0.30.1 k8s.io/client-go v0.30.1 @@ -119,9 +122,7 @@ require ( go.opentelemetry.io/otel/metric v1.27.0 // indirect go.opentelemetry.io/otel/trace v1.27.0 // indirect go.starlark.net v0.0.0-20240520160348-046347dcd104 // indirect - go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.23.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect diff --git a/go.sum b/go.sum index cf21a2d0..da02f96f 100644 --- a/go.sum +++ b/go.sum @@ -515,6 +515,8 @@ go.starlark.net v0.0.0-20240520160348-046347dcd104 h1:3qhteRISupnJvaWshOmeqEUs2y go.starlark.net v0.0.0-20240520160348-046347dcd104/go.mod h1:YKMCv9b1WrfWmeqdV5MAuEHWsu5iC+fe6kYl2sQjdI8= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= diff --git a/main.go b/main.go index b4100ba8..700ebb29 100644 --- a/main.go +++ b/main.go @@ -140,7 +140,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), DefaultCreationPolicy: creationPolicy, - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(ctx, mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "GitRepo") os.Exit(1) }