[nexus] Support Bundle background task

dev-tools/omdb/tests/env.out

@@ -162,6 +162,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "support_bundle_collector"
+    Manage support bundle collection and cleanup
+
+
 task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
@@ -333,6 +337,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "support_bundle_collector"
+    Manage support bundle collection and cleanup
+
+
 task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
@@ -491,6 +499,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "support_bundle_collector"
+    Manage support bundle collection and cleanup
+
+
 task: "switch_port_config_manager"
     manages switch port settings for rack switches
 

dev-tools/omdb/tests/successes.out

@@ -380,6 +380,10 @@ task: "service_zone_nat_tracker"
     ensures service zone nat records are recorded in NAT RPW table
 
 
+task: "support_bundle_collector"
+    Manage support bundle collection and cleanup
+
+
 task: "switch_port_config_manager"
     manages switch port settings for rack switches
 
@@ -693,6 +697,13 @@ task: "service_zone_nat_tracker"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: inventory collection is None
 
+task: "support_bundle_collector"
+  configured period: every <REDACTED_DURATION>s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    last completion reported error: task disabled
+
 task: "switch_port_config_manager"
   configured period: every <REDACTED_DURATION>s
   currently executing: no
@@ -1138,6 +1149,13 @@ task: "service_zone_nat_tracker"
     started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
     last completion reported error: inventory collection is None
 
+task: "support_bundle_collector"
+  configured period: every <REDACTED_DURATION>s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by a periodic timer firing
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    last completion reported error: task disabled
+
 task: "switch_port_config_manager"
   configured period: every <REDACTED_DURATION>s
   currently executing: no

nexus-config/src/nexus_config.rs

@@ -371,6 +371,8 @@ pub struct BackgroundTaskConfig {
     pub nat_cleanup: NatCleanupConfig,
     /// configuration for inventory tasks
     pub inventory: InventoryConfig,
+    /// configuration for support bundle collection
+    pub support_bundle_collector: SupportBundleCollectorConfig,
     /// configuration for physical disk adoption tasks
     pub physical_disk_adoption: PhysicalDiskAdoptionConfig,
     /// configuration for decommissioned disk cleaner task
@@ -458,6 +460,20 @@ pub struct ExternalEndpointsConfig {
     // allow/disallow wildcard certs, don't serve expired certs, etc.)
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct SupportBundleCollectorConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+
+    /// A toggle to disable support bundle collection
+    ///
+    /// Default: Off
+    #[serde(default)]
+    pub disable: bool,
+}
+
 #[serde_as]
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct PhysicalDiskAdoptionConfig {
@@ -931,6 +947,7 @@ mod test {
             inventory.period_secs = 10
             inventory.nkeep = 11
             inventory.disable = false
+            support_bundle_collector.period_secs = 30
             physical_disk_adoption.period_secs = 30
             decommissioned_disk_cleaner.period_secs = 30
             phantom_disks.period_secs = 30
@@ -1069,6 +1086,11 @@ mod test {
                             nkeep: 11,
                             disable: false,
                         },
+                        support_bundle_collector:
+                            SupportBundleCollectorConfig {
+                                period_secs: Duration::from_secs(30),
+                                disable: false,
+                            },
                         physical_disk_adoption: PhysicalDiskAdoptionConfig {
                             period_secs: Duration::from_secs(30),
                             disable: false,
@@ -1203,6 +1225,7 @@ mod test {
             inventory.period_secs = 10
             inventory.nkeep = 3
             inventory.disable = false
+            support_bundle_collector.period_secs = 30
             physical_disk_adoption.period_secs = 30
             decommissioned_disk_cleaner.period_secs = 30
             phantom_disks.period_secs = 30

nexus/Cargo.toml

@@ -42,6 +42,7 @@ hickory-resolver.workspace = true
 http.workspace = true
 http-body-util.workspace = true
 hyper.workspace = true
+hyper-staticfile.workspace = true
 illumos-utils.workspace = true
 internal-dns-resolver.workspace = true
 internal-dns-types.workspace = true
@@ -81,6 +82,7 @@ serde.workspace = true
 serde_json.workspace = true
 serde_urlencoded.workspace = true
 serde_with.workspace = true
+sha2.workspace = true
 sled-agent-client.workspace = true
 slog.workspace = true
 slog-async.workspace = true
@@ -119,6 +121,7 @@ update-common.workspace = true
 update-engine.workspace = true
 omicron-workspace-hack.workspace = true
 omicron-uuid-kinds.workspace = true
+zip.workspace = true
 
 [dev-dependencies]
 async-bb8-diesel.workspace = true

nexus/db-queries/src/db/datastore/mod.rs

@@ -123,6 +123,7 @@ pub use region::RegionAllocationParameters;
 pub use silo::Discoverability;
 pub use sled::SledTransition;
 pub use sled::TransitionError;
+pub use support_bundle::SupportBundleExpungementReport;
 pub use switch_port::SwitchPortSettingsCombinedResult;
 pub use virtual_provisioning_collection::StorageType;
 pub use vmm::VmmStateUpdateResult;

nexus/db-queries/src/db/datastore/support_bundle.rs

@@ -335,15 +335,6 @@ impl DataStore {
                             .execute_async(conn)
                             .await?;
 
-                    let Some(arbitrary_valid_nexus) =
-                        valid_nexus_zones.get(0).cloned()
-                    else {
-                        return Err(external::Error::internal_error(
-                            "No valid Nexuses, we cannot re-assign this support bundle",
-                        )
-                        .into());
-                    };
-
                     // Find all bundles on nexuses that no longer exist.
                     let bundles_with_bad_nexuses = dsl::support_bundle
                         .filter(dsl::assigned_nexus.eq_any(invalid_nexus_zones))
@@ -362,7 +353,7 @@ impl DataStore {
                             }
                         }).collect::<Vec<_>>();
 
-                    // Mark these support bundles as failing, and assign then
+                    // Mark these support bundles as failing, and assign them
                     // to a nexus that should still exist.
                     //
                     // This should lead to their storage being freed, if it
@@ -377,18 +368,41 @@ impl DataStore {
                         ))
                         .execute_async(conn)
                         .await?;
-                    let bundles_reassigned = diesel::update(dsl::support_bundle)
+
+                    let mut report = SupportBundleExpungementReport {
+                        bundles_failed_missing_datasets,
+                        bundles_deleted_missing_datasets,
+                        bundles_failing_missing_nexus,
+                        bundles_reassigned: 0,
+                    };
+
+                    // Exit a little early if there are no bundles to re-assign.
+                    //
+                    // This is a tiny optimization, but really, it means that
+                    // tests without Nexuses in their blueprints can succeed if
+                    // they also have no support bundles. In practice, this is
+                    // rare, but in our existing test framework, it's fairly
+                    // common.
+                    if bundles_to_reassign.is_empty() {
+                        return Ok(report);
+                    }
+
+                    let Some(arbitrary_valid_nexus) =
+                        valid_nexus_zones.get(0).cloned()
+                    else {
+                        return Err(external::Error::internal_error(
+                            "No valid Nexuses, we cannot re-assign this support bundle",
+                        )
+                        .into());
+                    };
+
+                    report.bundles_reassigned = diesel::update(dsl::support_bundle)
                         .filter(dsl::id.eq_any(bundles_to_reassign))
                         .set(dsl::assigned_nexus.eq(arbitrary_valid_nexus))
                         .execute_async(conn)
                         .await?;
 
-                    Ok(SupportBundleExpungementReport {
-                        bundles_failed_missing_datasets,
-                        bundles_deleted_missing_datasets,
-                        bundles_failing_missing_nexus,
-                        bundles_reassigned,
-                    })
+                    Ok(report)
                 }
                 .boxed()
             },

nexus/db-queries/src/db/datastore/zpool.rs

@@ -15,6 +15,7 @@ use crate::db::error::public_error_from_diesel;
 use crate::db::error::ErrorHandler;
 use crate::db::identity::Asset;
 use crate::db::model::PhysicalDisk;
+use crate::db::model::PhysicalDiskPolicy;
 use crate::db::model::PhysicalDiskState;
 use crate::db::model::Sled;
 use crate::db::model::Zpool;
@@ -31,9 +32,11 @@ use omicron_common::api::external::DataPageParams;
 use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
+use omicron_common::api::external::LookupResult;
 use omicron_common::api::external::LookupType;
 use omicron_common::api::external::ResourceType;
 use omicron_uuid_kinds::GenericUuid;
+use omicron_uuid_kinds::SledUuid;
 use omicron_uuid_kinds::ZpoolUuid;
 use uuid::Uuid;
 
@@ -270,4 +273,41 @@ impl DataStore {
 
         Ok(())
     }
+
+    pub async fn zpool_get_sled_if_in_service(
+        &self,
+        opctx: &OpContext,
+        id: ZpoolUuid,
+    ) -> LookupResult<SledUuid> {
+        opctx.authorize(authz::Action::ListChildren, &authz::FLEET).await?;
+        use db::schema::physical_disk::dsl as physical_disk_dsl;
+        use db::schema::zpool::dsl as zpool_dsl;
+
+        let conn = self.pool_connection_authorized(opctx).await?;
+        let id = zpool_dsl::zpool
+            .filter(zpool_dsl::id.eq(id.into_untyped_uuid()))
+            .filter(zpool_dsl::time_deleted.is_null())
+            .inner_join(
+                physical_disk_dsl::physical_disk
+                    .on(zpool_dsl::physical_disk_id.eq(physical_disk_dsl::id)),
+            )
+            .filter(
+                physical_disk_dsl::disk_policy
+                    .eq(PhysicalDiskPolicy::InService),
+            )
+            .select(zpool_dsl::sled_id)
+            .first_async::<Uuid>(&*conn)
+            .await
+            .map_err(|e| {
+                public_error_from_diesel(
+                    e,
+                    ErrorHandler::NotFoundByLookup(
+                        ResourceType::Zpool,
+                        LookupType::by_id(id.into_untyped_uuid()),
+                    ),
+                )
+            })?;
+
+        Ok(SledUuid::from_untyped_uuid(id))
+    }
 }

nexus/examples/config-second.toml

@@ -122,6 +122,7 @@ inventory.nkeep = 5
 inventory.disable = false
 phantom_disks.period_secs = 30
 physical_disk_adoption.period_secs = 30
+support_bundle_collector.period_secs = 30
 decommissioned_disk_cleaner.period_secs = 60
 blueprints.period_secs_load = 10
 blueprints.period_secs_execute = 60

nexus/examples/config.toml

@@ -108,6 +108,7 @@ inventory.nkeep = 5
 inventory.disable = false
 phantom_disks.period_secs = 30
 physical_disk_adoption.period_secs = 30
+support_bundle_collector.period_secs = 30
 decommissioned_disk_cleaner.period_secs = 60
 blueprints.period_secs_load = 10
 blueprints.period_secs_execute = 60

nexus/reconfigurator/execution/src/lib.rs

@@ -103,6 +103,13 @@ pub async fn realize_blueprint_with_overrides(
         blueprint,
     );
 
+    register_support_bundle_failure_step(
+        &engine.for_component(ExecutionComponent::SupportBundles),
+        &opctx,
+        datastore,
+        blueprint,
+    );
+
     let sled_list = register_sled_list_step(
         &engine.for_component(ExecutionComponent::SledList),
         &opctx,
@@ -243,6 +250,30 @@ fn register_zone_external_networking_step<'a>(
         .register();
 }
 
+fn register_support_bundle_failure_step<'a>(
+    registrar: &ComponentRegistrar<'_, 'a>,
+    opctx: &'a OpContext,
+    datastore: &'a DataStore,
+    blueprint: &'a Blueprint,
+) {
+    registrar
+        .new_step(
+            ExecutionStepId::Ensure,
+            "Mark support bundles as failed if they rely on an expunged disk or sled",
+            move |_cx| async move {
+                datastore
+                    .support_bundle_fail_expunged(
+                        &opctx, blueprint,
+                    )
+                    .await
+                    .map_err(|err| anyhow!(err))?;
+
+                StepSuccess::new(()).into()
+            },
+        )
+        .register();
+}
+
 fn register_sled_list_step<'a>(
     registrar: &ComponentRegistrar<'_, 'a>,
     opctx: &'a OpContext,