Unregister efrm-nondl on shutdown / reboot

On shutdown / reboot, if netdevs don't have a refcount of 1, the
kernel loops with message:
  unregister_netdevice: waiting for eth0 to become free. Usage count = 3
  unregister_netdevice: waiting for eth0 to become free. Usage count = 3
  unregister_netdevice: waiting for eth0 to become free. Usage count = 3
  [...]

With netdev refcount tracking, the traces are visible:
  unregister_netdevice: waiting for eth0 to become free. Usage count = 3
  ref_tracker: eth%d@ff3cf768c8344548 has 1/2 users at
       efrm_nic_add+0x29b/0x460 [sfc_resource]
       efrm_nondl_add_device+0x124/0x1c0 [sfc_resource]
       efrm_nondl_try_add_device+0x27/0xf0 [sfc_resource]
       efrm_nondl_register_netdev+0x106/0x160 [sfc_resource]
       nondl_register_store+0x174/0x1e0 [sfc_resource]
       kernfs_fop_write_iter+0x128/0x1c0
       vfs_write+0x308/0x420
       ksys_write+0x5f/0xe0
       do_syscall_64+0x7b/0x160
       entry_SYSCALL_64_after_hwframe+0x76/0x7e

  ref_tracker: eth%d@ff3cf768c8344548 has 1/2 users at
       efrm_nondl_register_netdev+0xa9/0x160 [sfc_resource]
       nondl_register_store+0x174/0x1e0 [sfc_resource]
       kernfs_fop_write_iter+0x128/0x1c0
       vfs_write+0x308/0x420
       ksys_write+0x5f/0xe0
       do_syscall_64+0x7b/0x160
       entry_SYSCALL_64_after_hwframe+0x76/0x7e

This patch makes sfc_resource register a reboot notifier that calls
efrm_nondl_unregister & efrm_nondl_shutdown that handles each of the two
refcounts. Considering that the only other caller of those functions
is the cleanup_sfc_resource module exit function, I don't think shutdown
should be concerned about module unload racing under normal circumstances.
In any case, efrm_nondl_unregister_driver is made to exit early before
the assert, in the event that the race does end up happening.

Signed-off-by: YiFei Zhu <zhuyifei@google.com>

src/driver/linux_resource/nondl_resource.c

@@ -95,6 +95,9 @@ void efrm_nondl_unregister_driver(struct efrm_nondl_driver *driver)
 
   rtnl_lock();
 
+  if (!nondl_driver)
+    goto out;
+
   EFRM_ASSERT(nondl_driver == driver);
 
   list_for_each_entry_safe_reverse(device, device_n, &driver->devices,
@@ -105,6 +108,7 @@ void efrm_nondl_unregister_driver(struct efrm_nondl_driver *driver)
 
   nondl_driver = NULL;
 
+out:
   rtnl_unlock();
 }
 EXPORT_SYMBOL(efrm_nondl_unregister_driver);

src/driver/linux_resource/resource_driver.c

@@ -69,6 +69,7 @@
 #include "sfcaffinity.h"
 #include <ci/driver/resource/linux_efhw_nic.h>
 #include <ci/driver/resource/driverlink.h>
+#include <linux/reboot.h>
 #include "debugfs.h"
 
 MODULE_AUTHOR("Solarflare Communications");
@@ -631,6 +632,21 @@ static void efrm_nic_del_all(void)
 		efrm_nic_del(linux_efhw_nic(nic));
 }
 
+static int sfc_resource_shutdown_notify(struct notifier_block *unused1,
+				        unsigned long unused2, void *unused3)
+{
+	/* Due to refcounting reasons in netdev, these must
+	 * be called for the shutdown to happen without delays
+	 */
+	efrm_nondl_unregister();
+	efrm_nondl_shutdown();
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block sfc_resource_shutdown_nb = {
+	.notifier_call = sfc_resource_shutdown_notify,
+};
 
 /****************************************************************************
  *
@@ -683,6 +699,8 @@ static int init_sfc_resource(void)
 	efrm_install_sysfs_entries();
 	efrm_nondl_register();
 
+	register_reboot_notifier(&sfc_resource_shutdown_nb);
+
 	return 0;
 
 failed_notifier:
@@ -705,6 +723,8 @@ static int init_sfc_resource(void)
  ****************************************************************************/
 static void cleanup_sfc_resource(void)
 {
+	unregister_reboot_notifier(&sfc_resource_shutdown_nb);
+
 	efrm_nondl_unregister();
 	efrm_remove_sysfs_entries();
 	efrm_nondl_shutdown();