diff --git a/.editorconfig b/.editorconfig index a7fa2f2..97e7097 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,16 +5,18 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true + [*.kt] ij_kotlin_call_parameters_new_line_after_left_paren = false ij_kotlin_call_parameters_right_paren_on_new_line = false ij_kotlin_method_parameters_new_line_after_left_paren = false ij_kotlin_method_parameters_right_paren_on_new_line = false ij_any_wrap_long_lines = false -ij_formatter_off_tag = "@formatter:off" -ij_formatter_on_tag = "@formatter:on" ij_kotlin_name_count_to_use_star_import = 50000 ij_kotlin_name_count_to_use_star_import_for_members = 50000 +ij_formatter_tags_enabled = true +ij_formatter_off_tag = @formatter:off +ij_formatter_on_tag = @formatter:on max_line_length = 256 indent_style = space indent_size = 4 diff --git a/docs/deployment/bases/clustered/operator/serviceaccount.yaml b/docs/deployment/bases/clustered/operator/serviceaccount.yaml index 6b79faf..b0ebe7b 100644 --- a/docs/deployment/bases/clustered/operator/serviceaccount.yaml +++ b/docs/deployment/bases/clustered/operator/serviceaccount.yaml @@ -35,6 +35,9 @@ rules: - apiGroups: [ "extensions" ] resources: [ "replicasets" ] verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ] + - apiGroups: [ "zalando.org" ] + resources: [ "routegroups" ] + verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/docs/deployment/bases/clustered/skipper-ds/kustomization.yaml b/docs/deployment/bases/clustered/skipper-ds/kustomization.yaml index 00afa62..b2bf127 100644 --- a/docs/deployment/bases/clustered/skipper-ds/kustomization.yaml +++ b/docs/deployment/bases/clustered/skipper-ds/kustomization.yaml @@ -4,3 +4,4 @@ resources: - serviceaccount.yaml - daemonset.yaml - service.yaml + - routegroups.crd.yaml diff --git a/docs/deployment/bases/clustered/skipper-ds/routegroups.crd.yaml b/docs/deployment/bases/clustered/skipper-ds/routegroups.crd.yaml new file mode 100644 index 0000000..cb2e62c --- /dev/null +++ b/docs/deployment/bases/clustered/skipper-ds/routegroups.crd.yaml @@ -0,0 +1,206 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + name: routegroups.zalando.org +spec: + group: zalando.org + names: + categories: + - all + kind: RouteGroup + listKind: RouteGroupList + plural: routegroups + shortNames: + - rg + - rgs + singular: routegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Hosts defined for the RouteGroup + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Address of the Load Balancer for the RouteGroup + jsonPath: .status.loadBalancer + name: Address + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + backends: + description: List of backends that can be referenced in the routes. + items: + properties: + address: + description: Address is required for Type network + type: string + algorithm: + description: Algorithm is required for Type lb + enum: + - roundRobin + - random + - consistentHash + - powerOfRandomNChoices + type: string + endpoints: + description: Endpoints is required for Type lb + items: + type: string + minItems: 1 + type: array + name: + description: Name is the BackendName that can be referenced as RouteGroupBackendReference + type: string + serviceName: + description: ServiceName is required for Type service + type: string + servicePort: + description: ServicePort is required for Type service + type: integer + type: + description: Type is one of "service|shunt|loopback|dynamic|lb|network" + enum: + - service + - shunt + - loopback + - dynamic + - lb + - network + type: string + required: + - name + - type + type: object + type: array + defaultBackends: + description: DefaultBackends is a list of default backends defined if no explicit backend is defined for a route. + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + hosts: + description: List of hostnames for the RouteGroup. + items: + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + type: string + type: array + routes: + items: + properties: + backends: + description: RouteGroupBackendReference specifies the list of backendReference that should be applied to override the defaultBackends + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + filters: + description: Filters specifies the list of filters applied to the routeSpec + items: + type: string + type: array + methods: + description: Methods defines valid HTTP methods for the specified routeSpec + items: + description: HTTPMethod is a valid HTTP method in uppercase. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path specifies Path predicate, only one of Path or PathSubtree is allowed + type: string + pathRegexp: + description: PathRegexp can be added additionally + type: string + pathSubtree: + description: PathSubtree specifies PathSubtree predicate, only one of Path or PathSubtree is allowed + type: string + predicates: + description: Predicates specifies the list of predicates applied to the routeSpec + items: + type: string + type: array + type: object + minItems: 1 + type: array + required: + - backends + type: object + status: + properties: + loadBalancer: + description: LoadBalancer is similar to ingress status, such that external-dns has the same style as in ingress + properties: + routegroup: + description: RouteGroup is similar to Ingress in ingress status.LoadBalancer. + items: + properties: + hostname: + description: Hostname is the hostname of the load balancer and is empty if IP is set + type: string + ip: + description: IP is the IP address of the load balancer and is empty if Hostname is set + type: string + type: object + type: array + required: + - routegroup + type: object + required: + - loadBalancer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/docs/deployment/bases/clustered/skipper-hpa/kustomization.yaml b/docs/deployment/bases/clustered/skipper-hpa/kustomization.yaml index 7f7cf20..c150587 100644 --- a/docs/deployment/bases/clustered/skipper-hpa/kustomization.yaml +++ b/docs/deployment/bases/clustered/skipper-hpa/kustomization.yaml @@ -5,3 +5,4 @@ resources: - deployment.yaml - service.yaml - hpa.yaml + - routegroups.crd.yaml diff --git a/docs/deployment/bases/clustered/skipper-hpa/routegroups.crd.yaml b/docs/deployment/bases/clustered/skipper-hpa/routegroups.crd.yaml new file mode 100644 index 0000000..cb2e62c --- /dev/null +++ b/docs/deployment/bases/clustered/skipper-hpa/routegroups.crd.yaml @@ -0,0 +1,206 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + name: routegroups.zalando.org +spec: + group: zalando.org + names: + categories: + - all + kind: RouteGroup + listKind: RouteGroupList + plural: routegroups + shortNames: + - rg + - rgs + singular: routegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Hosts defined for the RouteGroup + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Address of the Load Balancer for the RouteGroup + jsonPath: .status.loadBalancer + name: Address + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + backends: + description: List of backends that can be referenced in the routes. + items: + properties: + address: + description: Address is required for Type network + type: string + algorithm: + description: Algorithm is required for Type lb + enum: + - roundRobin + - random + - consistentHash + - powerOfRandomNChoices + type: string + endpoints: + description: Endpoints is required for Type lb + items: + type: string + minItems: 1 + type: array + name: + description: Name is the BackendName that can be referenced as RouteGroupBackendReference + type: string + serviceName: + description: ServiceName is required for Type service + type: string + servicePort: + description: ServicePort is required for Type service + type: integer + type: + description: Type is one of "service|shunt|loopback|dynamic|lb|network" + enum: + - service + - shunt + - loopback + - dynamic + - lb + - network + type: string + required: + - name + - type + type: object + type: array + defaultBackends: + description: DefaultBackends is a list of default backends defined if no explicit backend is defined for a route. + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + hosts: + description: List of hostnames for the RouteGroup. + items: + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + type: string + type: array + routes: + items: + properties: + backends: + description: RouteGroupBackendReference specifies the list of backendReference that should be applied to override the defaultBackends + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + filters: + description: Filters specifies the list of filters applied to the routeSpec + items: + type: string + type: array + methods: + description: Methods defines valid HTTP methods for the specified routeSpec + items: + description: HTTPMethod is a valid HTTP method in uppercase. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path specifies Path predicate, only one of Path or PathSubtree is allowed + type: string + pathRegexp: + description: PathRegexp can be added additionally + type: string + pathSubtree: + description: PathSubtree specifies PathSubtree predicate, only one of Path or PathSubtree is allowed + type: string + predicates: + description: Predicates specifies the list of predicates applied to the routeSpec + items: + type: string + type: array + type: object + minItems: 1 + type: array + required: + - backends + type: object + status: + properties: + loadBalancer: + description: LoadBalancer is similar to ingress status, such that external-dns has the same style as in ingress + properties: + routegroup: + description: RouteGroup is similar to Ingress in ingress status.LoadBalancer. + items: + properties: + hostname: + description: Hostname is the hostname of the load balancer and is empty if IP is set + type: string + ip: + description: IP is the IP address of the load balancer and is empty if Hostname is set + type: string + type: object + type: array + required: + - routegroup + type: object + required: + - loadBalancer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/docs/deployment/bases/namespaced/operator/serviceaccount.yaml b/docs/deployment/bases/namespaced/operator/serviceaccount.yaml index 5309747..a99d778 100644 --- a/docs/deployment/bases/namespaced/operator/serviceaccount.yaml +++ b/docs/deployment/bases/namespaced/operator/serviceaccount.yaml @@ -35,6 +35,9 @@ rules: - apiGroups: [ "extensions" ] resources: [ "replicasets" ] verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ] + - apiGroups: ["zalando.org" ] + resources: [ "routegroups" ] + verbs: [ "get", "list", "watch", "create", "update", "patch", "delete" ] --- # Make it possible for the operator to access the ShinyProxy namespace kind: RoleBinding diff --git a/docs/deployment/bases/namespaced/skipper-ds/kustomization.yaml b/docs/deployment/bases/namespaced/skipper-ds/kustomization.yaml index 00afa62..b2bf127 100644 --- a/docs/deployment/bases/namespaced/skipper-ds/kustomization.yaml +++ b/docs/deployment/bases/namespaced/skipper-ds/kustomization.yaml @@ -4,3 +4,4 @@ resources: - serviceaccount.yaml - daemonset.yaml - service.yaml + - routegroups.crd.yaml diff --git a/docs/deployment/bases/namespaced/skipper-ds/routegroups.crd.yaml b/docs/deployment/bases/namespaced/skipper-ds/routegroups.crd.yaml new file mode 100644 index 0000000..cb2e62c --- /dev/null +++ b/docs/deployment/bases/namespaced/skipper-ds/routegroups.crd.yaml @@ -0,0 +1,206 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + name: routegroups.zalando.org +spec: + group: zalando.org + names: + categories: + - all + kind: RouteGroup + listKind: RouteGroupList + plural: routegroups + shortNames: + - rg + - rgs + singular: routegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Hosts defined for the RouteGroup + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Address of the Load Balancer for the RouteGroup + jsonPath: .status.loadBalancer + name: Address + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + backends: + description: List of backends that can be referenced in the routes. + items: + properties: + address: + description: Address is required for Type network + type: string + algorithm: + description: Algorithm is required for Type lb + enum: + - roundRobin + - random + - consistentHash + - powerOfRandomNChoices + type: string + endpoints: + description: Endpoints is required for Type lb + items: + type: string + minItems: 1 + type: array + name: + description: Name is the BackendName that can be referenced as RouteGroupBackendReference + type: string + serviceName: + description: ServiceName is required for Type service + type: string + servicePort: + description: ServicePort is required for Type service + type: integer + type: + description: Type is one of "service|shunt|loopback|dynamic|lb|network" + enum: + - service + - shunt + - loopback + - dynamic + - lb + - network + type: string + required: + - name + - type + type: object + type: array + defaultBackends: + description: DefaultBackends is a list of default backends defined if no explicit backend is defined for a route. + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + hosts: + description: List of hostnames for the RouteGroup. + items: + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + type: string + type: array + routes: + items: + properties: + backends: + description: RouteGroupBackendReference specifies the list of backendReference that should be applied to override the defaultBackends + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + filters: + description: Filters specifies the list of filters applied to the routeSpec + items: + type: string + type: array + methods: + description: Methods defines valid HTTP methods for the specified routeSpec + items: + description: HTTPMethod is a valid HTTP method in uppercase. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path specifies Path predicate, only one of Path or PathSubtree is allowed + type: string + pathRegexp: + description: PathRegexp can be added additionally + type: string + pathSubtree: + description: PathSubtree specifies PathSubtree predicate, only one of Path or PathSubtree is allowed + type: string + predicates: + description: Predicates specifies the list of predicates applied to the routeSpec + items: + type: string + type: array + type: object + minItems: 1 + type: array + required: + - backends + type: object + status: + properties: + loadBalancer: + description: LoadBalancer is similar to ingress status, such that external-dns has the same style as in ingress + properties: + routegroup: + description: RouteGroup is similar to Ingress in ingress status.LoadBalancer. + items: + properties: + hostname: + description: Hostname is the hostname of the load balancer and is empty if IP is set + type: string + ip: + description: IP is the IP address of the load balancer and is empty if Hostname is set + type: string + type: object + type: array + required: + - routegroup + type: object + required: + - loadBalancer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/docs/deployment/bases/namespaced/skipper-hpa/kustomization.yaml b/docs/deployment/bases/namespaced/skipper-hpa/kustomization.yaml index 7f7cf20..c150587 100644 --- a/docs/deployment/bases/namespaced/skipper-hpa/kustomization.yaml +++ b/docs/deployment/bases/namespaced/skipper-hpa/kustomization.yaml @@ -5,3 +5,4 @@ resources: - deployment.yaml - service.yaml - hpa.yaml + - routegroups.crd.yaml diff --git a/docs/deployment/bases/namespaced/skipper-hpa/routegroups.crd.yaml b/docs/deployment/bases/namespaced/skipper-hpa/routegroups.crd.yaml new file mode 100644 index 0000000..cb2e62c --- /dev/null +++ b/docs/deployment/bases/namespaced/skipper-hpa/routegroups.crd.yaml @@ -0,0 +1,206 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.0 + name: routegroups.zalando.org +spec: + group: zalando.org + names: + categories: + - all + kind: RouteGroup + listKind: RouteGroupList + plural: routegroups + shortNames: + - rg + - rgs + singular: routegroup + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Hosts defined for the RouteGroup + jsonPath: .spec.hosts + name: Hosts + type: string + - description: Address of the Load Balancer for the RouteGroup + jsonPath: .status.loadBalancer + name: Address + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + backends: + description: List of backends that can be referenced in the routes. + items: + properties: + address: + description: Address is required for Type network + type: string + algorithm: + description: Algorithm is required for Type lb + enum: + - roundRobin + - random + - consistentHash + - powerOfRandomNChoices + type: string + endpoints: + description: Endpoints is required for Type lb + items: + type: string + minItems: 1 + type: array + name: + description: Name is the BackendName that can be referenced as RouteGroupBackendReference + type: string + serviceName: + description: ServiceName is required for Type service + type: string + servicePort: + description: ServicePort is required for Type service + type: integer + type: + description: Type is one of "service|shunt|loopback|dynamic|lb|network" + enum: + - service + - shunt + - loopback + - dynamic + - lb + - network + type: string + required: + - name + - type + type: object + type: array + defaultBackends: + description: DefaultBackends is a list of default backends defined if no explicit backend is defined for a route. + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + hosts: + description: List of hostnames for the RouteGroup. + items: + pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" + type: string + type: array + routes: + items: + properties: + backends: + description: RouteGroupBackendReference specifies the list of backendReference that should be applied to override the defaultBackends + items: + properties: + backendName: + description: BackendName references the skipperBackend by name + type: string + weight: + description: Weight defines the traffic weight, if there are 2 or more default backends + minimum: 0 + type: integer + required: + - backendName + type: object + type: array + filters: + description: Filters specifies the list of filters applied to the routeSpec + items: + type: string + type: array + methods: + description: Methods defines valid HTTP methods for the specified routeSpec + items: + description: HTTPMethod is a valid HTTP method in uppercase. + enum: + - GET + - HEAD + - POST + - PUT + - PATCH + - DELETE + - CONNECT + - OPTIONS + - TRACE + type: string + type: array + path: + description: Path specifies Path predicate, only one of Path or PathSubtree is allowed + type: string + pathRegexp: + description: PathRegexp can be added additionally + type: string + pathSubtree: + description: PathSubtree specifies PathSubtree predicate, only one of Path or PathSubtree is allowed + type: string + predicates: + description: Predicates specifies the list of predicates applied to the routeSpec + items: + type: string + type: array + type: object + minItems: 1 + type: array + required: + - backends + type: object + status: + properties: + loadBalancer: + description: LoadBalancer is similar to ingress status, such that external-dns has the same style as in ingress + properties: + routegroup: + description: RouteGroup is similar to Ingress in ingress status.LoadBalancer. + items: + properties: + hostname: + description: Hostname is the hostname of the load balancer and is empty if IP is set + type: string + ip: + description: IP is the IP address of the load balancer and is empty if Hostname is set + type: string + type: object + type: array + required: + - routegroup + type: object + required: + - loadBalancer + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pom.xml b/pom.xml index aec6f29..eb71e27 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ eu.openanalytics shinyproxy-operator - 1.1.0 + 1.2.0-SNAPSHOT Open Analytics NV diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt index d9da7b8..5c7b812 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/Operator.kt @@ -150,12 +150,12 @@ class Operator(client: NamespacedKubernetesClient? = null, replicaSetListener = ResourceListener(sendChannel, this.client.inAnyNamespace().apps().replicaSets()) serviceListener = ResourceListener(sendChannel, this.client.inAnyNamespace().services()) configMapListener = ResourceListener(sendChannel, this.client.inAnyNamespace().configMaps()) - ingressController = IngressController(sendChannel, this.client, this.client.inAnyNamespace().network().v1().ingresses()) + ingressController = IngressController(sendChannel, this.client.inAnyNamespace(), this.client.inAnyNamespace().network().v1().ingresses()) } else { replicaSetListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).apps().replicaSets()) serviceListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).services()) configMapListener = ResourceListener(sendChannel, this.client.inNamespace(namespace).configMaps()) - ingressController = IngressController(sendChannel, this.client, this.client.inNamespace(namespace).network().v1().ingresses()) + ingressController = IngressController(sendChannel, this.client.inNamespace(namespace), this.client.inNamespace(namespace).network().v1().ingresses()) } } @@ -164,39 +164,58 @@ class Operator(client: NamespacedKubernetesClient? = null, */ val shinyProxyController = ShinyProxyController(channel, this.client, shinyProxyClient, ingressController, podRetriever, reconcileListener) - fun prepare(): Pair> { - logger.info { "Starting background processes of ShinyProxy Operator" } + private fun _checkCrdExists(name: String, shortName: String) { try { - if (client.apiextensions().v1().customResourceDefinitions().withName("shinyproxies.openanalytics.eu").get() == null) { + if (client.apiextensions().v1().customResourceDefinitions().withName(name).get() == null) { println() println() - println("ERROR: the CustomResourceDefinition (CRD) of the Operator does not exist!") - println("The name of the CRD is 'shinyproxies.openanalytics.eu'") + println("ERROR: the CustomResourceDefinition (CRD) '${shortName}' does not exist!") + println("The name of the CRD is '${name}'") println("Create the CRD first, before starting the operator") println() println("Exiting in 10 seconds because of the above error") - Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by an sysadmin + Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by a sysadmin exitProcess(2) } } catch (e: KubernetesClientException) { println() println() - println("Warning: could not check whether ShinyProxy CRD exits.") + println("Warning: could not check whether $shortName CRD exits.") println("This is normal when the ServiceAccount of the operator does not have permission to access CRDs (at cluster scope).") println("If you get an unexpected error after this message, make sure that the CRD exists.") println() println() } - val shinyProxyLister = Lister(shinyProxyListener.start()) - val replicaSetLister = Lister(replicaSetListener.start(shinyProxyLister)) - val serviceLister = Lister(serviceListener.start(shinyProxyLister)) - val configMapLister = Lister(configMapListener.start(shinyProxyLister)) - val ingressLister = Lister(ingressController.start(shinyProxyLister)) - val resourceRetriever = ResourceRetriever(replicaSetLister, configMapLister, serviceLister, ingressLister) + } + + fun prepare(): Pair> { + logger.info { "Starting background processes of ShinyProxy Operator" } - return resourceRetriever to shinyProxyLister + _checkCrdExists("shinyproxies.openanalytics.eu", "ShinyProxy") + _checkCrdExists("routegroups.zalando.org", "RouteGroup") + + try { + val shinyProxyLister = Lister(shinyProxyListener.start()) + val replicaSetLister = Lister(replicaSetListener.start(shinyProxyLister)) + val serviceLister = Lister(serviceListener.start(shinyProxyLister)) + val configMapLister = Lister(configMapListener.start(shinyProxyLister)) + val ingressLister = Lister(ingressController.start(shinyProxyLister)) + val resourceRetriever = ResourceRetriever(replicaSetLister, configMapLister, serviceLister, ingressLister) + return resourceRetriever to shinyProxyLister + } catch (e: KubernetesClientException) { + println() + println() + println("Error during starting up. Please check if all CRDs exists (see above).") + println("Exiting in 10 seconds because of the above error") + println() + e.printStackTrace() + println() + println() + Thread.sleep(10000) // sleep 10 seconds to make it easier to find this error by a sysadmin + exitProcess(3) + } } suspend fun run(resourceRetriever: ResourceRetriever, shinyProxyLister: Lister) { diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt index aa9cf6f..f5704e0 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/LabelFactory.kt @@ -42,6 +42,13 @@ object LabelFactory { ) } + fun labelsForShinyProxy(shinyProxy: ShinyProxy): Map { + return mapOf( + APP_LABEL to APP_LABEL_VALUE, + NAME_LABEL to shinyProxy.metadata.name + ) + } + const val APP_LABEL = "app" const val APP_LABEL_VALUE = "shinyproxy" const val NAME_LABEL = "openanalytics.eu/sp-resource-name" diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt index 48d06fa..505eeff 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/components/ResourceNameFactory.kt @@ -46,8 +46,12 @@ object ResourceNameFactory { return "sp-${shinyProxy.metadata.name}-rs-${shinyProxyInstance.hashOfSpec}".take(KUBE_RESOURCE_NAME_MAX_LENGTH) } - fun createNameForIngress(shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): String { - return "sp-${shinyProxy.metadata.name}-ing-${shinyProxyInstance.hashOfSpec}".take(KUBE_RESOURCE_NAME_MAX_LENGTH) + fun createNameForIngress(shinyProxy: ShinyProxy, routeName: String, shinyProxyInstance: ShinyProxyInstance): String { + return listOf("sp", shinyProxy.metadata.name, "ing", routeName, shinyProxyInstance.hashOfSpec).filter { it.isNotEmpty() }.joinToString("-").take(KUBE_RESOURCE_NAME_MAX_LENGTH) + } + + fun createNameForMetadataIngress(shinyProxy: ShinyProxy): String { + return "sp-${shinyProxy.metadata.name}-ing-metadata".take(KUBE_RESOURCE_NAME_MAX_LENGTH) } } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt index 078cd6d..bf4ba40 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/controller/ShinyProxyController.kt @@ -172,6 +172,7 @@ class ShinyProxyController(private val channel: Channel, logger.debug { "${shinyProxy.logPrefix()} Trying to update status (attempt ${i}/5)" } tryUpdateStatus() logger.debug { "${shinyProxy.logPrefix()} Status successfully updated" } + ingressController.reconcileMetadataEndpoint(refreshShinyProxy(shinyProxy), true) return } catch (e: KubernetesClientException) { logger.warn(e) { "${shinyProxy.logPrefix()} Update of status not succeeded (attempt ${i}/5)" } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/crd/ShinyProxy.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/crd/ShinyProxy.kt index 61029e5..d5cfe46 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/crd/ShinyProxy.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/crd/ShinyProxy.kt @@ -131,7 +131,7 @@ class ShinyProxy : CustomResource(), Namespaced { return@lazy path } - return@lazy "" + return@lazy "/" } @get:JsonIgnore diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt index 6768843..7ac05da 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/IIngressController.kt @@ -31,4 +31,5 @@ interface IIngressController { fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) fun stop() + fun reconcileMetadataEndpoint(shinyProxy: ShinyProxy, force: Boolean = false) } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressController.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressController.kt index fe6860e..1d6773d 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressController.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressController.kt @@ -22,6 +22,7 @@ package eu.openanalytics.shinyproxyoperator.ingress.skipper import eu.openanalytics.shinyproxyoperator.components.LabelFactory import eu.openanalytics.shinyproxyoperator.components.LabelFactory.INGRESS_IS_LATEST +import eu.openanalytics.shinyproxyoperator.components.ResourceNameFactory import eu.openanalytics.shinyproxyoperator.controller.ResourceRetriever import eu.openanalytics.shinyproxyoperator.controller.ShinyProxyEvent import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy @@ -30,7 +31,7 @@ import eu.openanalytics.shinyproxyoperator.ingress.IIngressController import io.fabric8.kubernetes.api.model.apps.ReplicaSet import io.fabric8.kubernetes.api.model.networking.v1.Ingress import io.fabric8.kubernetes.api.model.networking.v1.IngressList -import io.fabric8.kubernetes.client.KubernetesClient +import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.fabric8.kubernetes.client.dsl.MixedOperation import io.fabric8.kubernetes.client.dsl.Resource import io.fabric8.kubernetes.client.informers.cache.Indexer @@ -41,7 +42,7 @@ import mu.KotlinLogging class IngressController( channel: SendChannel, - private val kubernetesClient: KubernetesClient, + private val kubernetesClient: NamespacedKubernetesClient, ingressClient: MixedOperation> ) : IIngressController { @@ -50,9 +51,12 @@ class IngressController( // Note: do not move this to the DiContainer since it is a Skipper-specific implementation private val ingressListener = IngressListener(channel, kubernetesClient, ingressClient) - + private val routeGroupClient = kubernetesClient.resources(RouteGroup::class.java) + private val metadataIngressFactory = MetadataRouteGroupFactory(routeGroupClient) + private val routeGroupListener = RouteGroupListener(this, routeGroupClient) fun start(shinyProxyLister: Lister): Indexer { + routeGroupListener.start(shinyProxyLister) return ingressListener.start(shinyProxyLister) } @@ -69,6 +73,7 @@ class IngressController( if (failed) { throw RuntimeException("One or more ingresses failed to reconcile") } + reconcileMetadataEndpoint(shinyProxy,false) } override fun onRemoveInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) { @@ -84,11 +89,13 @@ class IngressController( private fun reconcileSingleInstance(resourceRetriever: ResourceRetriever, shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance) { val ingresses = resourceRetriever.getIngressByLabels(LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance), shinyProxy.metadata.namespace) - val mustBeUpdated = if (ingresses.isEmpty()) { + val mustBeUpdated = if (ingresses.size < 3) { true } else { // if the label indicating this is the latest is different from the actual state -> reconcile - ingresses[0].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance + ingresses[0].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance || + ingresses[1].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance || + ingresses[2].metadata.labels[INGRESS_IS_LATEST]?.toBoolean() != shinyProxyInstance.isLatestInstance } if (mustBeUpdated) { @@ -116,4 +123,11 @@ class IngressController( return replicaSets[0] } + override fun reconcileMetadataEndpoint(shinyProxy: ShinyProxy, force: Boolean) { + val existingObject = routeGroupClient.inNamespace(shinyProxy.metadata.namespace).withName(ResourceNameFactory.createNameForMetadataIngress(shinyProxy)).get() + if (existingObject == null || force) { + metadataIngressFactory.createOrReplaceRouteGroup(shinyProxy) + } + } + } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressFactory.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressFactory.kt index 72768f3..b91b796 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressFactory.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressFactory.kt @@ -42,81 +42,91 @@ class IngressFactory(private val kubeClient: KubernetesClient) { val isLatest = shinyProxyInstance.isLatestInstance - val cookiePath = if (shinyProxy.subPath != "") { - shinyProxy.subPath - } else { - "/" - } + val routes = createRoutes(isLatest, hashOfSpec, shinyProxy) - val security = if (Operator.getOperatorInstance().disableSecureCookies) { - "" - } else { - "Secure;" + val labels = LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance).toMutableMap() + labels[LabelFactory.INGRESS_IS_LATEST] = isLatest.toString() + + for ((routeName, routeAnnotations) in routes) { + + //@formatter:off + val ingressDefinition = IngressBuilder() + .withNewMetadata() + .withName(ResourceNameFactory.createNameForIngress(shinyProxy, routeName, shinyProxyInstance)) + .withLabels(labels) + .addNewOwnerReference() + .withController(true) + .withKind("ReplicaSet") + .withApiVersion("apps/v1") + .withName(ResourceNameFactory.createNameForReplicaSet(shinyProxy, shinyProxyInstance)) + .withNewUid(replicaSet.metadata.uid) + .endOwnerReference() + .withAnnotations(routeAnnotations) + .endMetadata() + .withNewSpec() + .withIngressClassName("skipper") + .addNewRule() + .withHost(shinyProxy.fqdn) + .withNewHttp() + .addToPaths(createPathV1(shinyProxy, shinyProxyInstance)) + .endHttp() + .endRule() + .endSpec() + .build() + //@formatter:on + + val createdIngress = kubeClient.network().v1().ingresses().inNamespace(shinyProxy.metadata.namespace).createOrReplace(ingressDefinition) + logger.debug { "${shinyProxy.logPrefix(shinyProxyInstance)} [Component/Ingress] Created ${createdIngress.metadata.name} [latest=$isLatest]" } } - val annotations = if (isLatest) { - mapOf( - "zalando.org/skipper-predicate" to "True()", - "zalando.org/skipper-filter" to - """setRequestHeader("X-ShinyProxy-Instance", "$hashOfSpec")""" + - """ -> """ + - """setRequestHeader("X-ShinyProxy-Latest-Instance", "${shinyProxy.hashOfCurrentSpec}")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-instance=$hashOfSpec; $security Path=$cookiePath")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${shinyProxy.hashOfCurrentSpec}; $security Path=$cookiePath")""" + } + private fun createRoutes(isLatest: Boolean, hashOfSpec: String, shinyProxy: ShinyProxy): Map> { + return if (isLatest) { + mapOf( + "" to createRoute(true, hashOfSpec, shinyProxy, "True()"), + "cookie-override" to createRoute(false, hashOfSpec, shinyProxy, """Cookie("sp-instance-override", "$hashOfSpec") && Weight(20)"""), + "query-override" to createRoute(false, hashOfSpec, shinyProxy, """QueryParam("sp_instance_override", "$hashOfSpec") && Weight(20)"""), ) } else { mapOf( - "zalando.org/skipper-predicate" to """True() && Cookie("sp-instance", "$hashOfSpec")""", - "zalando.org/skipper-filter" to - """setRequestHeader("X-ShinyProxy-Instance", "$hashOfSpec")""" + - """ -> """ + - """setRequestHeader("X-ShinyProxy-Latest-Instance", "${shinyProxy.hashOfCurrentSpec}")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${shinyProxy.hashOfCurrentSpec}; $security Path=$cookiePath")""" + "" to createRoute(false, hashOfSpec, shinyProxy, """Cookie("sp-instance", "$hashOfSpec") && Weight(10)"""), + "cookie-override" to createRoute(false, hashOfSpec, shinyProxy, """Cookie("sp-instance-override", "$hashOfSpec") && Weight(20)"""), + "query-override" to createRoute(false, hashOfSpec, shinyProxy, """QueryParam("sp_instance_override", "$hashOfSpec") && Weight(20)"""), ) } + } - val labels = LabelFactory.labelsForShinyProxyInstance(shinyProxy, shinyProxyInstance).toMutableMap() - labels[LabelFactory.INGRESS_IS_LATEST] = isLatest.toString() + private fun createRoute(isDefaultRoute: Boolean, hashOfSpec: String, shinyProxy: ShinyProxy, predicate: String): Map { + val security = if (Operator.getOperatorInstance().disableSecureCookies) { + "" + } else { + "Secure;" + } - //@formatter:off - val ingressDefinition = IngressBuilder() - .withNewMetadata() - .withName(ResourceNameFactory.createNameForIngress(shinyProxy, shinyProxyInstance)) - .withLabels(labels) - .addNewOwnerReference() - .withController(true) - .withKind("ReplicaSet") - .withApiVersion("apps/v1") - .withName(ResourceNameFactory.createNameForReplicaSet(shinyProxy, shinyProxyInstance)) - .withNewUid(replicaSet.metadata.uid) - .endOwnerReference() - .withAnnotations(annotations) - .endMetadata() - .withNewSpec() - .addNewRule() - .withHost(shinyProxy.fqdn) - .withNewHttp() - .addToPaths(createPathV1(shinyProxy, shinyProxyInstance)) - .endHttp() - .endRule() - .withIngressClassName("skipper") - .endSpec() - .build() - //@formatter:on + return mapOf( + "zalando.org/skipper-predicate" to predicate, + "zalando.org/skipper-filter" to + """setRequestHeader("X-ShinyProxy-Instance", "$hashOfSpec")""" + + """ -> """ + + """setRequestHeader("X-ShinyProxy-Latest-Instance", "${shinyProxy.hashOfCurrentSpec}")""" + + if (isDefaultRoute) { + """ -> """ + + """appendResponseHeader("Set-Cookie", "sp-instance=$hashOfSpec; $security Path=${shinyProxy.subPath}")""" + } else { + "" + } + + """ -> """ + + """appendResponseHeader("Set-Cookie", "sp-latest-instance=${shinyProxy.hashOfCurrentSpec}; $security Path=${shinyProxy.subPath}")""", + ) - val createdIngress = - kubeClient.network().v1().ingresses().inNamespace(shinyProxy.metadata.namespace).createOrReplace(ingressDefinition) - logger.debug { "${shinyProxy.logPrefix(shinyProxyInstance)} [Component/Ingress] Created ${createdIngress.metadata.name} [latest=$isLatest]" } } private fun createPathV1 (shinyProxy: ShinyProxy, shinyProxyInstance: ShinyProxyInstance): HTTPIngressPath { //@formatter:off val builder = HTTPIngressPathBuilder() .withPathType("Prefix") + .withPath(shinyProxy.subPath) .withNewBackend() .withNewService() .withName(ResourceNameFactory.createNameForService(shinyProxy, shinyProxyInstance)) @@ -127,12 +137,6 @@ class IngressFactory(private val kubeClient: KubernetesClient) { .endBackend() //@formatter:on - if (shinyProxy.subPath != "") { - builder.withPath(shinyProxy.subPath) - } else { - builder.withPath("/") - } - return builder.build() } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressListener.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressListener.kt index 058fcd3..d808a17 100644 --- a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressListener.kt +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/IngressListener.kt @@ -77,7 +77,6 @@ class IngressListener(private val channel: SendChannel, private suspend fun enqueueResource(shinyProxyLister: Lister, trigger: String, resource: Ingress) { val replicaSetOwnerReference = getShinyProxyOwnerRefByKind(resource, "ReplicaSet") ?: return - // TODO namespace val replicaSet = kubernetesClient.apps().replicaSets().inNamespace(resource.metadata.namespace).withName(replicaSetOwnerReference.name).get() if (replicaSet == null) { logger.warn { "[${resource.kind}] [${resource.metadata.namespace}/${resource.metadata.name}] Cannot find owner (ReplicaSet) for this resource - probably the resource is being deleted" } diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/MetadataRouteGroupFactory.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/MetadataRouteGroupFactory.kt new file mode 100644 index 0000000..7bdf048 --- /dev/null +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/MetadataRouteGroupFactory.kt @@ -0,0 +1,82 @@ +/** + * ShinyProxy-Operator + * + * Copyright (C) 2021 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +package eu.openanalytics.shinyproxyoperator.ingress.skipper + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import eu.openanalytics.shinyproxyoperator.components.LabelFactory +import eu.openanalytics.shinyproxyoperator.components.ResourceNameFactory +import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy +import io.fabric8.kubernetes.api.model.KubernetesResourceList +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource +import mu.KotlinLogging + + +class MetadataRouteGroupFactory(private val routeGroupClient: MixedOperation, Resource>) { + + private val logger = KotlinLogging.logger {} + private val objectMapper = ObjectMapper().registerKotlinModule() + + fun createOrReplaceRouteGroup(shinyProxy: ShinyProxy) { + val metadata = objectMapper.writeValueAsString(mapOf("instances" to shinyProxy.status.instances)).replace("\"", "\\\"") + + val path = shinyProxy.subPath + "operator/metadata" + + val routeGroupSpec = RouteGroupSpec( + hosts = listOf(shinyProxy.fqdn), + backends = listOf(Backend("shunt", "shunt")), + defaultBackends = listOf(BackendName("shunt")), + routes = listOf( + Route( + pathSubtree = path, + filters = listOf( + """setResponseHeader("Content-Type","application/json")""", + """inlineContent("$metadata")""", + """status(200)""" + ), + backends = listOf(BackendName("shunt")) + ) + ) + ) + + //@formatter:off + val routeGroup = RouteGroup() + routeGroup.spec = routeGroupSpec + routeGroup.metadata = ObjectMetaBuilder() + .withNamespace(shinyProxy.metadata.namespace) + .withName(ResourceNameFactory.createNameForMetadataIngress(shinyProxy)) + .withLabels(LabelFactory.labelsForShinyProxy(shinyProxy)) + .addNewOwnerReference() + .withController(true) + .withKind("ShinyProxy") + .withApiVersion("openanalytics.eu/v1") + .withName(shinyProxy.metadata.name) + .withUid(shinyProxy.metadata.uid) + .endOwnerReference() + .build() + //@formatter:on + + val createdRouteGroup = routeGroupClient.inNamespace(shinyProxy.metadata.namespace).createOrReplace(routeGroup) + logger.info { "[Component/RouteGroup] Created ${createdRouteGroup.metadata.name}" } + } +} diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroup.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroup.kt new file mode 100644 index 0000000..c3eeb96 --- /dev/null +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroup.kt @@ -0,0 +1,40 @@ +/** + * ShinyProxy-Operator + * + * Copyright (C) 2021 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +package eu.openanalytics.shinyproxyoperator.ingress.skipper + +import io.fabric8.kubernetes.api.model.Namespaced +import io.fabric8.kubernetes.client.CustomResource +import io.fabric8.kubernetes.model.annotation.Group +import io.fabric8.kubernetes.model.annotation.Version + +class RouteGroupStatus() + +data class Backend(val name: String, val type: String) +data class BackendName(val backendName: String) +data class Route(val pathSubtree: String, val filters: List, val backends: List) +data class RouteGroupSpec(val hosts: List, val backends: List, val defaultBackends: List, val routes: List) + +@Version("v1") +@Group("zalando.org") +class RouteGroup: CustomResource(), Namespaced { + +} + diff --git a/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroupListener.kt b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroupListener.kt new file mode 100644 index 0000000..628de82 --- /dev/null +++ b/src/main/kotlin/eu/openanalytics/shinyproxyoperator/ingress/skipper/RouteGroupListener.kt @@ -0,0 +1,91 @@ +/** + * ShinyProxy-Operator + * + * Copyright (C) 2021 Open Analytics + * + * =========================================================================== + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Apache License as published by + * The Apache Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Apache License for more details. + * + * You should have received a copy of the Apache License + * along with this program. If not, see + */ +package eu.openanalytics.shinyproxyoperator.ingress.skipper + +import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy +import eu.openanalytics.shinyproxyoperator.isInManagedNamespace +import io.fabric8.kubernetes.api.model.HasMetadata +import io.fabric8.kubernetes.api.model.KubernetesResourceList +import io.fabric8.kubernetes.api.model.OwnerReference +import io.fabric8.kubernetes.client.dsl.MixedOperation +import io.fabric8.kubernetes.client.dsl.Resource +import io.fabric8.kubernetes.client.informers.ResourceEventHandler +import io.fabric8.kubernetes.client.informers.SharedIndexInformer +import io.fabric8.kubernetes.client.informers.cache.Indexer +import io.fabric8.kubernetes.client.informers.cache.Lister +import kotlinx.coroutines.runBlocking +import mu.KotlinLogging +import java.util.* + +// TODO this has some duplicate code with ResourceListener +class RouteGroupListener(private val ingressController: IngressController, + private val routeGroupClient: MixedOperation, Resource>) { + + private var informer: SharedIndexInformer? = null + private val logger = KotlinLogging.logger {} + + fun start(shinyProxyLister: Lister): Indexer? { + val i = routeGroupClient.inform(object : ResourceEventHandler { + override fun onAdd(resource: RouteGroup) { + logger.trace { "${resource.kind}::OnAdd ${resource.metadata.name}" } + runBlocking { enqueueResource(shinyProxyLister, "Add", resource) } + } + + override fun onUpdate(resource: RouteGroup, newResource: RouteGroup) { + logger.trace { "${resource.kind}::OnUpdate ${resource.metadata.name}" } + runBlocking { enqueueResource(shinyProxyLister, "Update", resource) } + } + + override fun onDelete(resource: RouteGroup, b: Boolean) { + logger.trace { "${resource.kind}::OnDelete ${resource.metadata.name}" } + runBlocking { enqueueResource(shinyProxyLister, "Delete", resource) } + } + }) + informer = i + return i.indexer + } + + fun stop() { + informer?.stop() + informer = null + } + + private fun enqueueResource(shinyProxyLister: Lister, trigger: String, resource: RouteGroup) { + val ownerReference = getShinyProxyOwnerRefByKind(resource, "ShinyProxy") ?: return + + val shinyProxy = shinyProxyLister.namespace(resource.metadata.namespace)[ownerReference.name] ?: return + if (!isInManagedNamespace(shinyProxy)) return + logger.debug { "${shinyProxy.logPrefix()} [Event/${trigger} component] [Component/${resource.kind}]" } + ingressController.reconcileMetadataEndpoint(shinyProxy) + } + + private fun getShinyProxyOwnerRefByKind(resource: HasMetadata, kind: String): OwnerReference? { + val ownerReferences = resource.metadata.ownerReferences + for (ownerReference in ownerReferences) { + if (ownerReference.kind.lowercase(Locale.getDefault()) == kind.lowercase(Locale.getDefault())) { + return ownerReference + } + } + + return null + } + +} diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt index 47dc623..81d8ad1 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/MainIntegrationTest.kt @@ -159,7 +159,7 @@ class MainIntegrationTest : IntegrationTestBase() { } // 3. wait until instance is created - spTestInstance.waitForOneReconcile() + spTestInstance.waitForReconcileCycle() logger.info { "Fully created instance." } // 4. assert correctness @@ -171,7 +171,7 @@ class MainIntegrationTest : IntegrationTestBase() { .withName("sp-${sp.metadata.name}-rs-${spTestInstance.hash}".take(63)).delete() logger.info { "Deleted ReplicaSet" } } - spTestInstance.waitForOneReconcile() + spTestInstance.waitForReconcileCycle() logger.info { "Reconciled after deleting RS" } spTestInstance.assertInstanceIsCorrect() @@ -201,7 +201,7 @@ class MainIntegrationTest : IntegrationTestBase() { .withName("sp-${sp.metadata.name}-ing-${spTestInstance.hash}".take(63)).delete() logger.info { "Deleted Ingress" } } - spTestInstance.waitForOneReconcile() + spTestInstance.waitForReconcileCycle() spTestInstance.assertInstanceIsCorrect() logger.info { "Reconciled after deleting Ingress" } @@ -550,11 +550,19 @@ class MainIntegrationTest : IntegrationTestBase() { spTestInstance.waitForOneReconcile() // 4. assert correctness - assertCorrectnessWithSubPath(spTestInstance, stableClient, namespace) + spTestInstance.assertInstanceIsCorrect(1, true) + + // 5. additional assert correctness of ingress + val ingresses = namespacedClient.inNamespace(namespace).network().v1().ingresses().list().items + assertEquals(3, ingresses.size) + for (ingress in ingresses) { + assertTrue(ingress.metadata.annotations["zalando.org/skipper-filter"]?.contains("Path=/sub-path/") == true) + } job.cancel() } + @Test fun `configuration with subpath ending in slash`() = setup(Mode.NAMESPACED) { namespace, shinyProxyClient, namespacedClient, stableClient, operator, reconcileListener -> @@ -578,136 +586,19 @@ class MainIntegrationTest : IntegrationTestBase() { spTestInstance.waitForOneReconcile() // 4. assert correctness - assertCorrectnessWithSubPath(spTestInstance, stableClient, namespace) - - job.cancel() - } - private fun assertCorrectnessWithSubPath( - spTestInstance: ShinyProxyTestInstance, - namespacedClient: NamespacedKubernetesClient, - namespace: String - ) { - val sp = spTestInstance.retrieveInstance() - assertNotNull(sp) - val instance = sp.status.instances.firstOrNull { it.hashOfSpec == spTestInstance.hash } - assertNotNull(instance) - assertEquals(true, instance.isLatestInstance) - assertEquals(1, sp.status.instances.size) + spTestInstance.assertInstanceIsCorrect(1, true) - // a. check configmap - spTestInstance.assertConfigMapIsCorrect(sp) - - // b. check replicaset - val replicaSets = namespacedClient.inNamespace(namespace).apps().replicaSets().list().items - assertEquals(1, replicaSets.size) - val replicaSet = - replicaSets.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash } - assertNotNull(replicaSet) - assertEquals(1, replicaSet.status.replicas) - assertEquals(1, replicaSet.status.readyReplicas) - assertEquals(1, replicaSet.status.availableReplicas) - assertEquals("sp-${sp.metadata.name}-rs-${spTestInstance.hash}".take(63), replicaSet.metadata.name) - spTestInstance.assertLabelsAreCorrect(replicaSet, sp) - spTestInstance.assertOwnerReferenceIsCorrect(replicaSet, sp) - - val templateSpec = replicaSet.spec.template.spec - assertEquals(1, templateSpec.containers.size) - assertEquals("shinyproxy", templateSpec.containers[0].name) - assertEquals(sp.image, templateSpec.containers[0].image) - assertEquals(sp.imagePullPolicy, templateSpec.containers[0].imagePullPolicy) - - assertEquals(3, templateSpec.containers[0].env.size) - assertNotNull(templateSpec.containers[0].env.firstOrNull { it.name == "SP_KUBE_POD_UID" }) - assertNotNull(templateSpec.containers[0].env.firstOrNull { it.name == "SP_KUBE_POD_NAME" }) - assertNotNull(templateSpec.containers[0].env.firstOrNull { it.name == "PROXY_REALM_ID" }) - assertEquals( - sp.metadata.name, - templateSpec.containers[0].env.firstOrNull { it.name == "PROXY_REALM_ID" }?.value - ) + // 5. additional assert correctness of ingress + val ingresses = namespacedClient.inNamespace(namespace).network().v1().ingresses().list().items + assertEquals(3, ingresses.size) + for (ingress in ingresses) { + assertTrue(ingress.metadata.annotations["zalando.org/skipper-filter"]?.contains("Path=/sub-path/") == true) + } - assertEquals(1, templateSpec.containers[0].volumeMounts.size) - assertEquals("config-volume", templateSpec.containers[0].volumeMounts[0].name) - assertEquals("/opt/shinyproxy/application.yml", templateSpec.containers[0].volumeMounts[0].mountPath) - assertEquals("application.yml", templateSpec.containers[0].volumeMounts[0].subPath) - - assertEquals(1, templateSpec.containers[0].livenessProbe.periodSeconds) - assertEquals("/actuator/health/liveness", templateSpec.containers[0].livenessProbe.httpGet.path) - assertEquals(IntOrString(9090), templateSpec.containers[0].livenessProbe.httpGet.port) - - assertEquals(1, templateSpec.containers[0].readinessProbe.periodSeconds) - assertEquals("/actuator/health/readiness", templateSpec.containers[0].readinessProbe.httpGet.path) - assertEquals(IntOrString(9090), templateSpec.containers[0].readinessProbe.httpGet.port) - - if (namespacedClient.isStartupProbesSupported()) { - // only check for startup probes if it supported - assertEquals(5, templateSpec.containers[0].startupProbe.periodSeconds) - assertEquals(6, templateSpec.containers[0].startupProbe.failureThreshold) - assertEquals("/actuator/health/liveness", templateSpec.containers[0].startupProbe.httpGet.path) - assertEquals(IntOrString(9090), templateSpec.containers[0].startupProbe.httpGet.port) + job.cancel() } - assertEquals(1, templateSpec.volumes.size) - assertEquals("config-volume", templateSpec.volumes[0].name) - assertEquals( - "sp-${sp.metadata.name}-cm-${spTestInstance.hash}".take(63), - templateSpec.volumes[0].configMap.name - ) - - assertTrue(Readiness.getInstance().isReady(replicaSet)) - - // c. check service - spTestInstance.assertServiceIsCorrect(sp) - - // d. check ingress - val ingresses = namespacedClient.inNamespace(namespace).network().v1().ingresses().list().items - assertEquals(1, ingresses.size) - val ingress = ingresses.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash } - assertNotNull(ingress) - assertEquals("sp-${sp.metadata.name}-ing-${spTestInstance.hash}".take(63), ingress.metadata.name) - - assertEquals( - mapOf( - LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE, - LabelFactory.NAME_LABEL to sp.metadata.name, - LabelFactory.INSTANCE_LABEL to spTestInstance.hash, - LabelFactory.INGRESS_IS_LATEST to "true" - ), ingress.metadata.labels - ) - - assertEquals(1, ingress.metadata.ownerReferences.size) - assertTrue(ingress.metadata.ownerReferences[0].controller) - assertEquals("ReplicaSet", ingress.metadata.ownerReferences[0].kind) - assertEquals("apps/v1", ingress.metadata.ownerReferences[0].apiVersion) - assertEquals( - "sp-${sp.metadata.name}-rs-${spTestInstance.hash}".take(63), - ingress.metadata.ownerReferences[0].name - ) - - assertEquals(ingress.spec.ingressClassName, "skipper") - - assertEquals(mapOf( - "zalando.org/skipper-predicate" to "True()", - "zalando.org/skipper-filter" to - """setRequestHeader("X-ShinyProxy-Instance", "${sp.hashOfCurrentSpec}")""" + - """ -> """ + - """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-instance=${sp.hashOfCurrentSpec}; Secure; Path=/sub-path/")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; Secure; Path=/sub-path/")""" - ), ingress.metadata.annotations) - - assertEquals(1, ingress.spec.rules.size) - val rule = ingress.spec.rules[0] - assertNotNull(rule) - assertEquals(sp.fqdn, rule.host) - assertEquals(1, rule.http.paths.size) - val path = rule.http.paths[0] - assertNotNull(path) - assertEquals("sp-${sp.metadata.name}-svc-${spTestInstance.hash}".take(63), path.backend.service.name) - assertEquals(80, path.backend.service.port.number) - } @Test fun `simple test namespaces non-secure cookie`() = setup( @@ -733,45 +624,15 @@ class MainIntegrationTest : IntegrationTestBase() { // 3. wait until instance is created spTestInstance.waitForOneReconcile() - // 4. assert correctness of ingress - val ingresses = namespacedClient.inNamespace(namespace).network().v1().ingresses().list().items - assertEquals(1, ingresses.size) - val ingress = ingresses.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == spTestInstance.hash } - assertNotNull(ingress) - assertEquals("sp-${sp.metadata.name}-ing-${spTestInstance.hash}".take(63), ingress.metadata.name) - - assertEquals( - mapOf( - LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE, - LabelFactory.NAME_LABEL to sp.metadata.name, - LabelFactory.INSTANCE_LABEL to spTestInstance.hash, - LabelFactory.INGRESS_IS_LATEST to "true" - ), ingress.metadata.labels - ) - - assertEquals(1, ingress.metadata.ownerReferences.size) - assertTrue(ingress.metadata.ownerReferences[0].controller) - assertEquals("ReplicaSet", ingress.metadata.ownerReferences[0].kind) - assertEquals("apps/v1", ingress.metadata.ownerReferences[0].apiVersion) - assertEquals( - "sp-${sp.metadata.name}-rs-${spTestInstance.hash}".take(63), - ingress.metadata.ownerReferences[0].name - ) - - assertEquals(ingress.spec.ingressClassName, "skipper") - - assertEquals(mapOf( - "zalando.org/skipper-predicate" to "True()", - "zalando.org/skipper-filter" to - """setRequestHeader("X-ShinyProxy-Instance", "${sp.hashOfCurrentSpec}")""" + - """ -> """ + - """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-instance=${sp.hashOfCurrentSpec}; Path=/")""" + - """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; Path=/")""" - ), ingress.metadata.annotations) + // 4. assert correctness + spTestInstance.assertInstanceIsCorrect(1, true) + // 5. additional assert correctness of ingress + val ingresses = namespacedClient.inNamespace(namespace).network().v1().ingresses().list().items + assertEquals(3, ingresses.size) + for (ingress in ingresses) { + assertTrue(ingress.metadata.annotations["zalando.org/skipper-filter"]?.contains("Secure;") == false) + } job.cancel() } @@ -1072,7 +933,7 @@ class MainIntegrationTest : IntegrationTestBase() { // B) at this point the ingress should exist ingresses = stableClient.inNamespace(namespace).network().v1().ingresses().list().items - assertEquals(1, ingresses.size) + assertEquals(3, ingresses.size) } diff --git a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/helpers/ShinyProxyTestInstance.kt b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/helpers/ShinyProxyTestInstance.kt index b44ab4b..8e28742 100644 --- a/src/test/kotlin/eu/openanalytics/shinyproxyoperator/helpers/ShinyProxyTestInstance.kt +++ b/src/test/kotlin/eu/openanalytics/shinyproxyoperator/helpers/ShinyProxyTestInstance.kt @@ -20,15 +20,23 @@ */ package eu.openanalytics.shinyproxyoperator.helpers +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.registerKotlinModule +import eu.openanalytics.shinyproxyoperator.Operator import eu.openanalytics.shinyproxyoperator.ShinyProxyClient import eu.openanalytics.shinyproxyoperator.components.LabelFactory import eu.openanalytics.shinyproxyoperator.crd.ShinyProxy import eu.openanalytics.shinyproxyoperator.crd.ShinyProxyInstance +import eu.openanalytics.shinyproxyoperator.crd.ShinyProxyStatus +import eu.openanalytics.shinyproxyoperator.ingress.skipper.RouteGroup import io.fabric8.kubernetes.api.model.HasMetadata import io.fabric8.kubernetes.api.model.IntOrString import io.fabric8.kubernetes.client.NamespacedKubernetesClient import io.fabric8.kubernetes.client.internal.readiness.Readiness +import kotlinx.coroutines.TimeoutCancellationException import kotlinx.coroutines.withTimeout +import java.nio.file.Paths +import kotlin.io.path.pathString import kotlin.test.assertEquals import kotlin.test.assertNotEquals import kotlin.test.assertNotNull @@ -41,6 +49,7 @@ class ShinyProxyTestInstance(private val namespace: String, private val reconcileListener: ReconcileListener) { lateinit var hash: String + private val objectMapper = ObjectMapper().registerKotlinModule() fun create(): ShinyProxy { val sp: ShinyProxy = shinyProxyClient.inNamespace(namespace).load(this.javaClass.getResourceAsStream("/configs/$fileName")).createOrReplace() @@ -51,12 +60,28 @@ class ShinyProxyTestInstance(private val namespace: String, return sp } - suspend fun waitForOneReconcile(): ShinyProxyInstance? { + suspend fun waitForOneReconcile(): ShinyProxyInstance { return withTimeout(120_000) { reconcileListener.waitForNextReconcile(hash).await() } } + suspend fun waitForReconcileCycle() { + // require at least one reconcile + waitForOneReconcile() + // then wait until no reconciles happen + while (true) { + try { + withTimeout(10_000) { + reconcileListener.waitForNextReconcile(hash).await() + } + } catch (e: TimeoutCancellationException) { + // no reconcile in the last 10 seconds -> ok + return + } + } + } + fun assertInstanceIsCorrect(numInstancesRunning: Int = 1, isLatest: Boolean = true) { val sp = retrieveInstance() assertNotNull(sp) @@ -76,29 +101,90 @@ class ShinyProxyTestInstance(private val namespace: String, // check ingress assertIngressIsCorrect(sp, numInstancesRunning, isLatest) - } - fun assertIngressIsCorrect(sp: ShinyProxy, numInstancesRunning: Int = 1, isLatest: Boolean = true) { - val ingresses = client.inNamespace(namespace).network().v1().ingresses().list().items - assertEquals(numInstancesRunning, ingresses.size) - val ingress = ingresses.firstOrNull { it.metadata.labels[LabelFactory.INSTANCE_LABEL] == hash } - assertNotNull(ingress) - assertEquals("sp-${sp.metadata.name}-ing-${hash}".take(63), ingress.metadata.name) + // check metadata RouteGroup + assertMetadataRouteGroupIsCorrect(sp) + } + private fun assertMetadataRouteGroupIsCorrect(sp: ShinyProxy) { + val routeGroupClient = client.resources(RouteGroup::class.java) + val routeGroups = routeGroupClient.inNamespace(namespace).withLabel(LabelFactory.NAME_LABEL, sp.metadata.name).list().items + assertEquals(1, routeGroups.size) + val routeGroup = routeGroups[0] + assertEquals("sp-${sp.metadata.name}-ing-metadata".take(63), routeGroup.metadata.name) assertEquals(mapOf( LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE, - LabelFactory.NAME_LABEL to sp.metadata.name, - LabelFactory.INSTANCE_LABEL to hash, - LabelFactory.INGRESS_IS_LATEST to isLatest.toString() - ), ingress.metadata.labels) + LabelFactory.NAME_LABEL to sp.metadata.name + ), routeGroup.metadata.labels) + + assertEquals(1, routeGroup.metadata.ownerReferences.size) + assertTrue(routeGroup.metadata.ownerReferences[0].controller) + assertEquals("ShinyProxy", routeGroup.metadata.ownerReferences[0].kind) + assertEquals("openanalytics.eu/v1", routeGroup.metadata.ownerReferences[0].apiVersion) + assertEquals(sp.metadata.name, routeGroup.metadata.ownerReferences[0].name) + + assertEquals(1, routeGroup.spec.hosts.size) + assertEquals(sp.fqdn, routeGroup.spec.hosts[0]) + + assertEquals(1, routeGroup.spec.backends.size) + assertEquals("shunt", routeGroup.spec.backends[0].name) + assertEquals("shunt", routeGroup.spec.backends[0].type) + + assertEquals(1, routeGroup.spec.defaultBackends.size) + assertEquals("shunt", routeGroup.spec.defaultBackends[0].backendName) + + assertEquals(1, routeGroup.spec.routes.size) + assertEquals(Paths.get(sp.subPath, "/operator/metadata").pathString, routeGroup.spec.routes[0].pathSubtree) + assertEquals(1, routeGroup.spec.routes[0].backends.size) + assertEquals("shunt", routeGroup.spec.routes[0].backends[0].backendName) + assertEquals(3, routeGroup.spec.routes[0].filters.size) + assertEquals("""setResponseHeader("Content-Type","application/json")""", routeGroup.spec.routes[0].filters[0]) + val metadata = routeGroup.spec.routes[0].filters[1].removePrefix("inlineContent(\"").removeSuffix("\")").replace("\\\"", "\"") + val status = objectMapper.readValue(metadata, ShinyProxyStatus::class.java) + assertEquals(sp.status, status) + assertEquals("""status(200)""", routeGroup.spec.routes[0].filters[2]) - assertEquals(1, ingress.metadata.ownerReferences.size) - assertTrue(ingress.metadata.ownerReferences[0].controller) - assertEquals("ReplicaSet", ingress.metadata.ownerReferences[0].kind) - assertEquals("apps/v1", ingress.metadata.ownerReferences[0].apiVersion) - assertEquals("sp-${sp.metadata.name}-rs-${hash}".take(63), ingress.metadata.ownerReferences[0].name) + } + + fun assertIngressIsCorrect(sp: ShinyProxy, numInstancesRunning: Int = 1, isLatest: Boolean = true) { + val allIngresses = client.inNamespace(namespace).network().v1().ingresses().list().items + assertEquals(numInstancesRunning * 3, allIngresses.size) + val ingresses = client.inNamespace(namespace).network().v1().ingresses().withLabel(LabelFactory.INSTANCE_LABEL, hash).list().items + assertEquals(3, ingresses.size) + val mainIngress = ingresses.firstOrNull { it.metadata.name == "sp-${sp.metadata.name}-ing-${hash}".take(63) } + assertNotNull(mainIngress) + val cookieOverrideIngress = ingresses.firstOrNull { it.metadata.name == "sp-${sp.metadata.name}-ing-cookie-override-${hash}".take(63) } + assertNotNull(cookieOverrideIngress) + val queryParamOverrideIngress = ingresses.firstOrNull { it.metadata.name == "sp-${sp.metadata.name}-ing-query-override-${hash}".take(63) } + assertNotNull(queryParamOverrideIngress) + + for (ingress in listOf(mainIngress, cookieOverrideIngress, queryParamOverrideIngress)) { + assertEquals(mapOf( + LabelFactory.APP_LABEL to LabelFactory.APP_LABEL_VALUE, + LabelFactory.NAME_LABEL to sp.metadata.name, + LabelFactory.INSTANCE_LABEL to hash, + LabelFactory.INGRESS_IS_LATEST to isLatest.toString() + ), ingress.metadata.labels) + + assertEquals(1, ingress.metadata.ownerReferences.size) + assertTrue(ingress.metadata.ownerReferences[0].controller) + assertEquals("ReplicaSet", ingress.metadata.ownerReferences[0].kind) + assertEquals("apps/v1", ingress.metadata.ownerReferences[0].apiVersion) + assertEquals("sp-${sp.metadata.name}-rs-${hash}".take(63), ingress.metadata.ownerReferences[0].name) + assertEquals(ingress.spec.ingressClassName, "skipper") + } + + val security = if (Operator.getOperatorInstance().disableSecureCookies) { + "" + } else { + "Secure;" + } + val cookiePath = if (sp.subPath != "") { + sp.subPath + } else { + "/" + } - assertEquals(ingress.spec.ingressClassName, "skipper") if (isLatest) { assertEquals(mapOf( @@ -108,32 +194,51 @@ class ShinyProxyTestInstance(private val namespace: String, """ -> """ + """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-instance=${sp.hashOfCurrentSpec}; Secure; Path=/")""" + + """appendResponseHeader("Set-Cookie", "sp-instance=${sp.hashOfCurrentSpec}; $security Path=$cookiePath")""" + """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; Secure; Path=/")""" - ), ingress.metadata.annotations) + """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; $security Path=$cookiePath")""" + ), mainIngress.metadata.annotations) } else { assertEquals(mapOf( - "zalando.org/skipper-predicate" to """True() && Cookie("sp-instance", "$hash")""", + "zalando.org/skipper-predicate" to """Cookie("sp-instance", "$hash") && Weight(10)""", "zalando.org/skipper-filter" to """setRequestHeader("X-ShinyProxy-Instance", "$hash")""" + """ -> """ + """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + """ -> """ + - """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; Secure; Path=/")""" - - ), ingress.metadata.annotations) + """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; $security Path=$cookiePath")""" + ), mainIngress.metadata.annotations) + } + assertEquals(mapOf( + "zalando.org/skipper-predicate" to """Cookie("sp-instance-override", "$hash") && Weight(20)""", + "zalando.org/skipper-filter" to + """setRequestHeader("X-ShinyProxy-Instance", "$hash")""" + + """ -> """ + + """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + + """ -> """ + + """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; $security Path=$cookiePath")""", + ), cookieOverrideIngress.metadata.annotations) + assertEquals(mapOf( + "zalando.org/skipper-predicate" to """QueryParam("sp_instance_override", "$hash") && Weight(20)""", + "zalando.org/skipper-filter" to + """setRequestHeader("X-ShinyProxy-Instance", "$hash")""" + + """ -> """ + + """setRequestHeader("X-ShinyProxy-Latest-Instance", "${sp.hashOfCurrentSpec}")""" + + """ -> """ + + """appendResponseHeader("Set-Cookie", "sp-latest-instance=${sp.hashOfCurrentSpec}; $security Path=$cookiePath")""" + ), queryParamOverrideIngress.metadata.annotations) + + for (ingress in listOf(mainIngress, cookieOverrideIngress, queryParamOverrideIngress)) { + assertEquals(1, ingress.spec.rules.size) + val rule = ingress.spec.rules[0] + assertNotNull(rule) + assertEquals(sp.fqdn, rule.host) + assertEquals(1, rule.http.paths.size) + val path = rule.http.paths[0] + assertNotNull(path) + assertEquals("sp-${sp.metadata.name}-svc-${hash}".take(63), path.backend.service.name) + assertEquals(80, path.backend.service.port.number) } - - assertEquals(1, ingress.spec.rules.size) - val rule = ingress.spec.rules[0] - assertNotNull(rule) - assertEquals(sp.fqdn, rule.host) - assertEquals(1, rule.http.paths.size) - val path = rule.http.paths[0] - assertNotNull(path) - assertEquals("sp-${sp.metadata.name}-svc-${hash}".take(63), path.backend.service.name) - assertEquals(80, path.backend.service.port.number) }