[sled-agent] Use zone-network-setup service after underlay is up (#7260)

This commit removes the need to zlogin into the switch zone to set the
underlay IP address and new static address after the underlay is up.
Instead it adds the new values to the SMF properties and refreshes the
zone-network-setup service. This service is safe to rerun as all it's
commands ensure things are there, but does not throw an error if they
are.

Closes: #6157

illumos-utils/src/smf_helper.rs

@@ -165,6 +165,38 @@ impl<'t> SmfHelper<'t> {
         Ok(())
     }
 
+    pub fn delpropvalue<P, V>(&self, prop: P, val: V) -> Result<(), Error>
+    where
+        P: ToString,
+        V: ToString,
+    {
+        match self
+            .running_zone
+            .run_cmd(&[
+                SVCCFG,
+                "-s",
+                &self.smf_name,
+                "delpropvalue",
+                &prop.to_string(),
+                &val.to_string(),
+            ])
+            .map_err(|err| Error::ZoneCommand {
+                intent: format!("del {} smf property value", prop.to_string()),
+                err,
+            }) {
+            Ok(_) => (),
+            Err(e) => {
+                // If a property already doesn't exist we don't need to
+                // return an error
+                if !e.to_string().contains("No such property") {
+                    return Err(e);
+                }
+            }
+        };
+
+        Ok(())
+    }
+
     pub fn delpropvalue_default_instance<P, V>(
         &self,
         prop: P,

sled-agent/src/services.rs

@@ -54,7 +54,6 @@ use illumos_utils::running_zone::{
 };
 use illumos_utils::smf_helper::SmfHelper;
 use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT;
-use illumos_utils::zone::AddressRequest;
 use illumos_utils::zpool::{PathInPool, ZpoolName};
 use illumos_utils::{execute, PFEXEC};
 use internal_dns_resolver::Resolver;
@@ -542,6 +541,18 @@ struct SwitchZoneConfigLocal {
     root: Utf8PathBuf,
 }
 
+/// Service that sets up common networking across zones
+struct ZoneNetworkSetupService {}
+
+impl illumos_utils::smf_helper::Service for ZoneNetworkSetupService {
+    fn service_name(&self) -> String {
+        "zone-network-setup".to_string()
+    }
+    fn smf_name(&self) -> String {
+        format!("svc:/oxide/{}", self.service_name())
+    }
+}
+
 /// Describes either an Omicron-managed zone or the switch zone, used for
 /// functions that operate on either one or the other
 enum ZoneArgs<'a> {
@@ -4287,59 +4298,53 @@ impl ServiceManager {
                 );
                 *request = new_request;
 
+                // Add SMF properties here and restart zone-network-setup service
                 let first_address = request.addresses.get(0);
                 let address = first_address
                     .map(|addr| addr.to_string())
                     .unwrap_or_else(|| "".to_string());
 
-                for addr in &request.addresses {
-                    if *addr == Ipv6Addr::LOCALHOST {
-                        continue;
-                    }
-                    info!(
-                        self.inner.log,
-                        "Ensuring address {} exists",
-                        addr.to_string()
-                    );
-                    let addr_request =
-                        AddressRequest::new_static(IpAddr::V6(*addr), None);
-                    zone.ensure_address(addr_request).await?;
-                    info!(
-                        self.inner.log,
-                        "Ensuring address {} exists - OK",
-                        addr.to_string()
-                    );
-                }
+                // Set new properties for the network set up service and refresh
+                let nw_setup_svc = ZoneNetworkSetupService {};
+                let nsmfh = SmfHelper::new(&zone, &nw_setup_svc);
+
+                nsmfh.delpropvalue("config/gateway", "*")?;
+                nsmfh.delpropvalue("config/static_addr", "*")?;
 
-                // When the request addresses have changed this means the underlay is
-                // available now as well.
                 if let Some(info) = self.inner.sled_info.get() {
-                    info!(
+                    nsmfh.addpropvalue_type(
+                        "config/gateway",
+                        &info.underlay_address,
+                        "astring",
+                    )?;
+                } else {
+                    // It should be impossible for the `sled_info` not to be set here.
+                    // When the request addresses have changed this means the underlay is
+                    // available as well.
+                    error!(
                         self.inner.log,
-                        "Ensuring there is a default route";
-                        "gateway" => ?info.underlay_address,
+                        concat!(
+                            "sled agent info is not present,",
+                            " even though underlay address exists"
+                        )
                     );
-                    match zone.add_default_route(info.underlay_address).map_err(
-                        |err| Error::ZoneCommand {
-                            intent: "Adding Route".to_string(),
-                            err,
-                        },
-                    ) {
-                        Ok(_) => (),
-                        Err(e) => {
-                            if e.to_string().contains("entry exists") {
-                                info!(
-                                    self.inner.log,
-                                    "Default route already exists";
-                                    "gateway" => ?info.underlay_address,
-                                )
-                            } else {
-                                return Err(e);
-                            }
-                        }
-                    };
                 }
 
+                for address in &request.addresses {
+                    if *address != Ipv6Addr::LOCALHOST {
+                        nsmfh.addpropvalue_type(
+                            "config/static_addr",
+                            &address,
+                            "astring",
+                        )?;
+                    }
+                }
+                nsmfh.refresh()?;
+                info!(
+                    self.inner.log,
+                    "refreshed zone-network-setup service with new configuration"
+                );
+
                 for service in &request.services {
                     let smfh = SmfHelper::new(&zone, service);
 

smf/zone-network-setup/manifest.xml

@@ -12,7 +12,7 @@
   <service_fmri value='svc:/milestone/network:default' />
   </dependency>
 
-  <!-- Run after the NDP daemon is online.. -->
+  <!-- Run after the NDP daemon is online. -->
   <dependency name='ndp' grouping='require_all' restart_on='none'
     type='service'>
   <service_fmri value='svc:/network/routing/ndp:default' />
@@ -31,6 +31,10 @@
     timeout_seconds='0' />
 
   <exec_method type='method' name='stop' exec=':true' timeout_seconds='0' />
+  <!-- We use the same command as the start method as it is safe to rerun. -->
+  <exec_method type='method' name='refresh'
+    exec='/opt/oxide/zone-setup-cli/bin/zone-setup common-networking -d %{config/datalink} -s %{config/static_addr} -g %{config/gateway}'
+    timeout_seconds='0' />
 
   <property_group name='startd' type='framework'>
     <propval name='duration' type='astring' value='transient' />