From aa5858c05607385f830201af400ceb2d0884e517 Mon Sep 17 00:00:00 2001 From: Danil-Grigorev Date: Tue, 21 May 2024 18:31:38 +0200 Subject: [PATCH] Add option for using agent initiated setup Signed-off-by: Danil-Grigorev --- config/rbac/role.yaml | 1 + justfile | 27 +++-- src/api/capi_cluster.rs | 3 +- src/api/capi_clusterclass.rs | 3 +- src/api/fleet_addon_config.rs | 34 +++++-- src/api/fleet_cluster.rs | 3 +- src/api/fleet_cluster_registration_token.rs | 30 ++++++ src/api/fleet_clustergroup.rs | 3 +- src/api/mod.rs | 2 + src/controllers/cluster.rs | 105 ++++++++++++++------ src/controllers/mod.rs | 3 + 11 files changed, 162 insertions(+), 52 deletions(-) create mode 100644 src/api/fleet_cluster_registration_token.rs diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index d2eed6b..1c1dd55 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -75,6 +75,7 @@ rules: resources: - clusters - clustergroups + - clusterregistrationtokens verbs: - create - get diff --git a/justfile b/justfile index f886689..10068aa 100644 --- a/justfile +++ b/justfile @@ -19,8 +19,8 @@ default: @just --list --unsorted --color=always # Generates stuff -generate: - just generate-addon-crds +generate features="": + just generate-addon-crds {{features}} just generate-crds # generates files for CRDS @@ -29,13 +29,14 @@ generate-crds: _create-out-dir _install-kopium _download-yq just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/kubernetes-sigs/cluster-api/main/config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml" "src/api/capi_clusterclass.rs" "" just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster.rs" "select(.spec.names.singular==\"cluster\")" "--no-condition" just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_clustergroup.rs" "select(.spec.names.singular==\"clustergroup\")" "--no-condition" + just _generate-kopium-url {{KOPIUM_BIN}} "https://raw.githubusercontent.com/rancher/fleet/main/charts/fleet-crd/templates/crds.yaml" "src/api/fleet_cluster_registration_token.rs" "select(.spec.names.singular==\"clusterregistrationtoken\")" "" [private] _generate-kopium-url kpath="" source="" dest="" yqexp="." condition="": curl -sSL {{source}} | {{YQ_BIN}} '{{yqexp}}' | {{kpath}} -D Default {{condition}} -f - > {{dest}} -generate-addon-crds: - cargo run --bin crdgen > config/crds/fleet-addon-config.yaml +generate-addon-crds features="": + cargo run --features={{features}} --bin crdgen > config/crds/fleet-addon-config.yaml # run with opentelemetry run-telemetry: @@ -70,7 +71,9 @@ _build features="": docker build -t {{ORG}}/{{NAME}}:{{TAG}} . # docker build base -build-base: (_build "") +build-base features="": + just _build {{features}} + # docker build with telemetry build-otel: (_build "telemetry") @@ -78,7 +81,8 @@ build-otel: (_build "telemetry") docker-push: docker push {{ORG}}/{{NAME}}:{{TAG}} -load-base: build-base +load-base features="": + just build-base {{features}} kind load docker-image {{ORG}}/{{NAME}}:{{TAG}} --name dev # Start local dev environment @@ -131,7 +135,9 @@ install-capi: _download-clusterctl EXP_CLUSTER_RESOURCE_SET=true CLUSTER_TOPOLOGY=true {{CLUSTERCTL_BIN}} init -i docker # Deploy will deploy the operator -deploy: _download-kustomize load-base +deploy features="": _download-kustomize + just generate {{features}} + just load-base {{features}} {{KUSTOMIZE_BIN}} build config/default | kubectl apply -f - undeploy: _download-kustomize @@ -146,7 +152,10 @@ test-import: start-dev deploy deploy-child-cluster deploy-crs kubectl wait clusters.fleet.cattle.io --timeout=300s --for=condition=Ready docker-demo # Full e2e test of importing cluster in fleet -test-cluster-class-import: start-dev deploy deploy-child-cluster-class deploy-crs +test-cluster-class-import: start-dev + just deploy "agent-initiated" + just deploy-child-cluster-class + just deploy-crs kubectl wait pods --for=condition=Ready --timeout=300s --all --all-namespaces kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=jsonpath='{.status.clusterCount}=1' quick-start kubectl wait clustergroups.fleet.cattle.io --timeout=300s --for=condition=Ready quick-start @@ -155,7 +164,7 @@ test-cluster-class-import: start-dev deploy deploy-child-cluster-class deploy-cr # Install kopium [private] _install-kopium: - cargo install --git https://github.com/kube-rs/kopium.git --branch main --root {{OUT_DIR}} + cargo install --git https://github.com/kube-rs/kopium.git --tag 0.19.0 --root {{OUT_DIR}} # Download kustomize [private] diff --git a/src/api/capi_cluster.rs b/src/api/capi_cluster.rs index f745c9b..d9e8777 100644 --- a/src/api/capi_cluster.rs +++ b/src/api/capi_cluster.rs @@ -1,6 +1,6 @@ // WARNING: generated by kopium - manual changes will be overwritten // kopium command: kopium -D Default -f - -// kopium version: 0.18.0 +// kopium version: 0.19.0 #[allow(unused_imports)] mod prelude { @@ -17,6 +17,7 @@ use self::prelude::*; #[kube(namespaced)] #[kube(status = "ClusterStatus")] #[kube(schema = "disabled")] +#[kube(derive="Default")] pub struct ClusterSpec { #[serde(default, skip_serializing_if = "Option::is_none", rename = "clusterNetwork")] pub cluster_network: Option, diff --git a/src/api/capi_clusterclass.rs b/src/api/capi_clusterclass.rs index 2ed2cef..b7fa64e 100644 --- a/src/api/capi_clusterclass.rs +++ b/src/api/capi_clusterclass.rs @@ -1,6 +1,6 @@ // WARNING: generated by kopium - manual changes will be overwritten // kopium command: kopium -D Default -f - -// kopium version: 0.18.0 +// kopium version: 0.19.0 #[allow(unused_imports)] mod prelude { @@ -17,6 +17,7 @@ use self::prelude::*; #[kube(namespaced)] #[kube(status = "ClusterClassStatus")] #[kube(schema = "disabled")] +#[kube(derive="Default")] pub struct ClusterClassSpec { #[serde(default, skip_serializing_if = "Option::is_none", rename = "controlPlane")] pub control_plane: Option, diff --git a/src/api/fleet_addon_config.rs b/src/api/fleet_addon_config.rs index c77a574..f93e372 100644 --- a/src/api/fleet_addon_config.rs +++ b/src/api/fleet_addon_config.rs @@ -24,14 +24,8 @@ impl Default for FleetAddonConfig { metadata: Default::default(), spec: FleetAddonConfigSpec { patch_resource: Some(true), - cluster_class: Some(ClusterClassConfig { - set_owner_references: Some(true), - enabled: Some(true), - }), - cluster: Some(ClusterConfig { - set_owner_references: Some(true), - enabled: Some(true), - }), + cluster_class: Some(ClusterClassConfig::default()), + cluster: Some(ClusterConfig::default()), }, } } @@ -48,6 +42,15 @@ pub struct ClusterClassConfig { pub set_owner_references: Option, } +impl Default for ClusterClassConfig { + fn default() -> Self { + Self { + set_owner_references: Some(true), + enabled: Some(true), + } + } +} + #[derive(Serialize, Deserialize, Clone, Debug, JsonSchema)] pub struct ClusterConfig { /// Enable Cluster config funtionality. @@ -59,4 +62,19 @@ pub struct ClusterConfig { /// Setting to disable setting owner references on the created resources pub set_owner_references: Option, + + #[cfg(feature = "agent-initiated")] + /// Prepare initial cluster for agent initiated connection + pub agent_initiated: Option, +} + +impl Default for ClusterConfig { + fn default() -> Self { + Self { + set_owner_references: Some(true), + enabled: Some(true), + #[cfg(feature = "agent-initiated")] + agent_initiated: Some(true), + } + } } diff --git a/src/api/fleet_cluster.rs b/src/api/fleet_cluster.rs index 8b45c27..637dfec 100644 --- a/src/api/fleet_cluster.rs +++ b/src/api/fleet_cluster.rs @@ -1,6 +1,6 @@ // WARNING: generated by kopium - manual changes will be overwritten // kopium command: kopium -D Default --no-condition -f - -// kopium version: 0.18.0 +// kopium version: 0.19.0 #[allow(unused_imports)] mod prelude { @@ -16,6 +16,7 @@ use self::prelude::*; #[kube(namespaced)] #[kube(status = "ClusterStatus")] #[kube(schema = "disabled")] +#[kube(derive="Default")] pub struct ClusterSpec { #[serde(default, skip_serializing_if = "Option::is_none", rename = "agentAffinity")] pub agent_affinity: Option, diff --git a/src/api/fleet_cluster_registration_token.rs b/src/api/fleet_cluster_registration_token.rs new file mode 100644 index 0000000..ebb3dce --- /dev/null +++ b/src/api/fleet_cluster_registration_token.rs @@ -0,0 +1,30 @@ +// WARNING: generated by kopium - manual changes will be overwritten +// kopium command: kopium -D Default -f - +// kopium version: 0.19.0 + +#[allow(unused_imports)] +mod prelude { + pub use kube::CustomResource; + pub use serde::{Serialize, Deserialize}; +} +use self::prelude::*; + +#[derive(CustomResource, Serialize, Deserialize, Clone, Debug, Default)] +#[kube(group = "fleet.cattle.io", version = "v1alpha1", kind = "ClusterRegistrationToken", plural = "clusterregistrationtokens")] +#[kube(namespaced)] +#[kube(status = "ClusterRegistrationTokenStatus")] +#[kube(schema = "disabled")] +#[kube(derive="Default")] +pub struct ClusterRegistrationTokenSpec { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttl: Option, +} + +#[derive(Serialize, Deserialize, Clone, Debug, Default)] +pub struct ClusterRegistrationTokenStatus { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expires: Option, + #[serde(default, skip_serializing_if = "Option::is_none", rename = "secretName")] + pub secret_name: Option, +} + diff --git a/src/api/fleet_clustergroup.rs b/src/api/fleet_clustergroup.rs index 9bcc5e4..8d1cd23 100644 --- a/src/api/fleet_clustergroup.rs +++ b/src/api/fleet_clustergroup.rs @@ -1,6 +1,6 @@ // WARNING: generated by kopium - manual changes will be overwritten // kopium command: kopium -D Default --no-condition -f - -// kopium version: 0.18.0 +// kopium version: 0.19.0 #[allow(unused_imports)] mod prelude { @@ -15,6 +15,7 @@ use self::prelude::*; #[kube(namespaced)] #[kube(status = "ClusterGroupStatus")] #[kube(schema = "disabled")] +#[kube(derive="Default")] pub struct ClusterGroupSpec { #[serde(default, skip_serializing_if = "Option::is_none")] pub selector: Option, diff --git a/src/api/mod.rs b/src/api/mod.rs index c12a5d1..a660271 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -3,3 +3,5 @@ pub mod capi_clusterclass; pub mod fleet_cluster; pub mod fleet_clustergroup; pub mod fleet_addon_config; +#[cfg(feature = "agent-initiated")] +pub mod fleet_cluster_registration_token; diff --git a/src/controllers/cluster.rs b/src/controllers/cluster.rs index f5ee22b..4b16c0b 100644 --- a/src/controllers/cluster.rs +++ b/src/controllers/cluster.rs @@ -3,11 +3,16 @@ use crate::api::capi_cluster::{Cluster, ClusterTopology}; use crate::api::fleet_addon_config::{ClusterConfig, FleetAddonConfig}; use crate::api::fleet_cluster; +#[cfg(feature = "agent-initiated")] +use crate::api::fleet_cluster_registration_token::{ + ClusterRegistrationToken, ClusterRegistrationTokenSpec, +}; use crate::Result; -use k8s_openapi::apimachinery::pkg::apis::meta::v1::OwnerReference; use kube::api::ObjectMeta; use kube::{api::ResourceExt, runtime::controller::Action, Resource}; +#[cfg(feature = "agent-initiated")] +use rand::distributions::{Alphanumeric, DistString as _}; use std::sync::Arc; @@ -19,43 +24,78 @@ pub static CONTROLPLANE_READY_CONDITION: &str = "ControlPlaneReady"; pub struct FleetClusterBundle { fleet: fleet_cluster::Cluster, + #[cfg(feature = "agent-initiated")] + cluster_registration_token: Option, config: FleetAddonConfig, } -impl From<&Cluster> for fleet_cluster::Cluster { +impl From<&Cluster> for ObjectMeta { fn from(cluster: &Cluster) -> Self { - let labels = match &cluster.spec.topology { + Self { + name: Some(cluster.name_any()), + namespace: cluster.meta().namespace.clone(), + ..Default::default() + } + } +} + +impl Cluster { + fn to_cluster(self: &Cluster, config: Option) -> fleet_cluster::Cluster { + let config = config.unwrap_or_default(); + let labels = match &self.spec.topology { Some(ClusterTopology { class, .. }) if !class.is_empty() => { - let mut labels = cluster.labels().clone(); + let mut labels = self.labels().clone(); labels.insert(CLUSTER_CLASS_LABEL.to_string(), class.clone()); labels } - None | Some(ClusterTopology { .. }) => cluster.labels().clone(), + None | Some(ClusterTopology { .. }) => self.labels().clone(), }; - Self { + fleet_cluster::Cluster { metadata: ObjectMeta { labels: Some(labels), - name: Some(cluster.name_any()), - namespace: cluster.meta().namespace.clone(), - owner_references: cluster - .controller_owner_ref(&()) - .into_iter() - .map(|r| OwnerReference { - controller: None, - ..r - }) - .map(Into::into) - .collect(), - ..Default::default() + owner_references: config + .set_owner_references + .is_some_and(|set| set) + .then_some(self.owner_ref(&()).into_iter().collect()), + ..self.into() }, + #[cfg(feature = "agent-initiated")] + spec: match config.agent_initiated { + Some(true) => fleet_cluster::ClusterSpec { + client_id: Some(Alphanumeric.sample_string(&mut rand::thread_rng(), 64)), + ..Default::default() + }, + None | Some(false) => fleet_cluster::ClusterSpec { + kube_config_secret: Some(format!("{}-kubeconfig", self.name_any())), + ..Default::default() + }, + }, + #[cfg(not(feature = "agent-initiated"))] spec: fleet_cluster::ClusterSpec { - kube_config_secret: Some(format!("{}-kubeconfig", cluster.name_any())), + kube_config_secret: Some(format!("{}-kubeconfig", self.name_any())), ..Default::default() }, status: Default::default(), } } + + #[cfg(feature = "agent-initiated")] + fn to_cluster_registration_token( + self: &Cluster, + config: Option, + ) -> Option { + config?.agent_initiated?.then_some(true)?; + + ClusterRegistrationToken { + metadata: self.into(), + spec: ClusterRegistrationTokenSpec { + ttl: Some("1h".into()), + }, + ..Default::default() + } + .into() + } } impl FleetBundle for FleetClusterBundle { @@ -66,12 +106,22 @@ impl FleetBundle for FleetClusterBundle { .map_err(Into::::into)?; if let Some(true) = self.config.spec.patch_resource { - patch(ctx, self.fleet.clone()) + patch(ctx.clone(), self.fleet.clone()) .await .map_err(Into::::into) .map_err(Into::::into)?; } + #[cfg(feature = "agent-initiated")] + get_or_create( + ctx, + self.cluster_registration_token + .clone() + .ok_or(SyncError::EarlyReturn)?, + ) + .await + .map_err(Into::::into)?; + Ok(Action::await_change()) } } @@ -90,18 +140,11 @@ impl FleetController for Cluster { self.cluster_ready().ok_or(SyncError::EarlyReturn)?; - let mut fleet: fleet_cluster::Cluster = self.into(); - if let Some(ClusterConfig { - set_owner_references: Some(true), - .. - }) = config.spec.cluster - { - } else { - fleet.metadata.owner_references = None - } - Ok(FleetClusterBundle { - fleet, + fleet: self.to_cluster(config.spec.cluster.clone()), + #[cfg(feature = "agent-initiated")] + cluster_registration_token: self + .to_cluster_registration_token(config.spec.cluster.clone()), config: config.clone(), }) } diff --git a/src/controllers/mod.rs b/src/controllers/mod.rs index 5be3adf..20da64e 100644 --- a/src/controllers/mod.rs +++ b/src/controllers/mod.rs @@ -8,6 +8,9 @@ pub enum SyncError { #[error("{0}")] GroupSync(#[from] GroupSyncError), + #[error("Cluster registration token create error {0}")] + ClusterRegistrationTokenSync(#[from] GetOrCreateError), + #[error("Return early")] EarlyReturn, }