Add fuse reliable check during reset

During board reset, the fuse entries will be removed and re-created.
Check whether the old entry still exists or not and add a delay to
repeat the check.

Signed-off-by: Liming Sun <lsun@mellanox.com>

src/rshim.c

@@ -30,7 +30,7 @@
 #define RSHIM_TIMER_INTERVAL 1
 
 /* Cycles to poll the network initialization before timeout. */
-#define RSHIM_NET_INIT_DELAY (30000 / RSHIM_TIMER_INTERVAL)
+#define RSHIM_NET_INIT_DELAY (60000 / RSHIM_TIMER_INTERVAL)
 
 /* Keepalive period in milliseconds. */
 static int rshim_keepalive_period = 300;
@@ -603,22 +603,22 @@ int rshim_boot_open(rshim_backend_t *bd)
   if (rc && rc != -EPROTO && rc != -ESHUTDOWN &&
     rc != -ETIMEDOUT && rc != -EPIPE) {
     RSHIM_ERR("boot_open: error %d writing reset control\n", rc);
-    pthread_mutex_unlock(&bd->mutex);
     bd->is_boot_open = 0;
+    pthread_mutex_unlock(&bd->mutex);
     return rc;
   }
 
   if (rc)
     RSHIM_ERR("boot_open: got error %d on reset write\n", rc);
 
 boot_open_done:
+  rshim_ref(bd);
   pthread_mutex_unlock(&bd->mutex);
 
+  /* A delay to workaround to wait for PCIe backend ready. */
   if (!bd->has_reprobe)
     sleep(10);
 
-  rshim_ref(bd);
-
   return 0;
 }
 
@@ -1553,6 +1553,8 @@ static int rshim_boot_done(rshim_backend_t *bd)
 
 int rshim_fifo_fsync(rshim_backend_t *bd, int chan)
 {
+  int rc = 0;
+
   pthread_mutex_lock(&bd->mutex);
 
   /*
@@ -1562,31 +1564,31 @@ int rshim_fifo_fsync(rshim_backend_t *bd, int chan)
    */
   while (!write_empty(bd, chan)) {
     if (pthread_cond_wait(&bd->write_fifo[chan].operable, &bd->mutex)) {
-      pthread_mutex_unlock(&bd->mutex);
-      return -EINTR;
+      rc = -EINTR;
+      break;
     }
 
     if (rshim_got_peer_signal() == 0) {
-      pthread_mutex_unlock(&bd->mutex);
-      return -EINTR;
+      rc = -EINTR;
+      break;
     }
   }
 
-  while (bd->spin_flags & RSH_SFLG_WRITING) {
+  while (!rc && (bd->spin_flags & RSH_SFLG_WRITING)) {
     if (pthread_cond_wait(&bd->fifo_write_complete_cond, &bd->mutex)) {
-      pthread_mutex_unlock(&bd->mutex);
-      return -EINTR;
+      rc = -EINTR;
+      break;
     }
 
     if (rshim_got_peer_signal() == 0) {
-      pthread_mutex_unlock(&bd->mutex);
-      return -EINTR;
+      rc = -EINTR;
+      break;
     }
   }
 
   pthread_mutex_unlock(&bd->mutex);
 
-  return 0;
+  return rc;
 }
 
 void rshim_fifo_check_poll(rshim_backend_t *bd, int chan, bool *poll_rx,

src/rshim_fuse.c

@@ -21,6 +21,7 @@
 #ifdef __linux__
 #include <fuse/cuse_lowlevel.h>
 #include <fuse/fuse_opt.h>
+#include <unistd.h>
 #elif defined(__FreeBSD__)
 #include <termios.h>
 #include <sys/stat.h>
@@ -1099,6 +1100,7 @@ int rshim_fuse_init(rshim_backend_t *bd)
 {
   char buf[128], *name;
 #ifdef __linux__
+  time_t t0, t1;
   const char *bufp[] = {buf};
   struct cuse_info ci = {.dev_info_argc = 1,
                          .dev_info_argv = bufp,
@@ -1124,6 +1126,21 @@ int rshim_fuse_init(rshim_backend_t *bd)
   for (i = 0; i < RSH_DEV_TYPES; i++) {
 #ifdef __linux__
     name = rshim_dev_minor_names[i];
+
+    /*
+     * Check whether path already exists. Adding a loop in case the
+     * device was re-ceated during SW_RESET.
+     */
+    snprintf(buf, sizeof(buf), "/dev/rshim%d/%s",
+             bd->index + rshim_index_base, name);
+    time(&t0);
+    while (!access(buf, F_OK)) {
+      time(&t1);
+      if (difftime(t1, t0) > 5) {
+        RSHIM_ERR("%s already exists\n", buf);
+        return -1;
+      }
+    }
     snprintf(buf, sizeof(buf), "DEVNAME=rshim%d/%s",
              bd->index + rshim_index_base, name);
     if (!ops[i])

src/rshim_pcie.c

@@ -333,7 +333,6 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
     dev = calloc(1, sizeof(*dev));
     if (dev == NULL) {
       ret = -ENOMEM;
-      rshim_unlock();
       goto error;
     }
 
@@ -350,8 +349,6 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
 
   rshim_ref(bd);
 
-  rshim_unlock();
-
   /* Initialize object */
   dev->pci_dev = pci_dev;
 
@@ -427,28 +424,25 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
    * has already registered or not, which involves reading/writting rshim
    * registers and has assumption that the under layer is working.
    */
-  rshim_lock();
   ret = rshim_register(bd);
   if (ret) {
-    rshim_unlock();
     pthread_mutex_unlock(&bd->mutex);
     goto rshim_map_failed;
   }
-  rshim_unlock();
 
   /* Notify that the device is attached */
   ret = rshim_notify(bd, RSH_EVENT_ATTACH, 0);
   pthread_mutex_unlock(&bd->mutex);
   if (ret)
     goto rshim_map_failed;
 
+  rshim_unlock();
   return 0;
 
  rshim_map_failed:
-   rshim_lock();
    rshim_deref(bd);
-   rshim_unlock();
  error:
+   rshim_unlock();
    return ret;
 }
 

src/rshim_pcie_lf.c

@@ -499,7 +499,6 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
     dev = calloc(1, sizeof(*dev));
     if (dev == NULL) {
       ret = -ENOMEM;
-      rshim_unlock();
       goto error;
     }
 
@@ -516,8 +515,6 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
 
   rshim_ref(bd);
 
-  rshim_unlock();
-
   /* Initialize object */
   dev->pci_dev = pci_dev;
 
@@ -528,28 +525,25 @@ static int rshim_pcie_probe(struct pci_dev *pci_dev)
    * has already registered or not, which involves reading/writting rshim
    * registers and has assumption that the under layer is working.
    */
-  rshim_lock();
   ret = rshim_register(bd);
   if (ret) {
-    rshim_unlock();
     pthread_mutex_unlock(&bd->mutex);
     goto rshim_map_failed;
   }
-  rshim_unlock();
 
   /* Notify that the device is attached */
   ret = rshim_notify(bd, RSH_EVENT_ATTACH, 0);
   pthread_mutex_unlock(&bd->mutex);
   if (ret)
     goto rshim_map_failed;
 
+  rshim_unlock();
   return 0;
 
 rshim_map_failed:
-  rshim_lock();
   rshim_deref(bd);
-  rshim_unlock();
 error:
+  rshim_unlock();
    return ret;
 }
 

src/rshim_usb.c

@@ -570,7 +570,6 @@ static int rshim_usb_probe_one(libusb_context *ctx, libusb_device *usb_dev)
     dev = calloc(1, sizeof(*dev));
     if (dev == NULL) {
       RSHIM_ERR("couldn't get memory for new device");
-      rshim_unlock();
       goto error;
     }
 
@@ -604,12 +603,9 @@ static int rshim_usb_probe_one(libusb_context *ctx, libusb_device *usb_dev)
 
   if (!dev->read_or_intr_urb || !dev->write_urb) {
     RSHIM_ERR("can't allocate buffers or urbs\n");
-    rshim_unlock();
     goto error;
   }
 
-  rshim_unlock();
-
   pthread_mutex_lock(&bd->mutex);
 
   for (i = 0; i < config->bNumInterfaces; i++) {
@@ -693,21 +689,19 @@ static int rshim_usb_probe_one(libusb_context *ctx, libusb_device *usb_dev)
    * has already registered or not, which involves reading/writting rshim
    * registers and has assumption that the under layer is working.
    */
-  rshim_lock();
   rc = rshim_register(bd);
   if (rc) {
-    rshim_unlock();
     pthread_mutex_unlock(&bd->mutex);
     goto error;
   }
-  rshim_unlock();
 
   /* Notify that device is attached. */
   rc = rshim_notify(bd, RSH_EVENT_ATTACH, 0);
   pthread_mutex_unlock(&bd->mutex);
   if (rc)
     goto error;
 
+  rshim_unlock();
   return 0;
 
 error:
@@ -721,11 +715,10 @@ static int rshim_usb_probe_one(libusb_context *ctx, libusb_device *usb_dev)
     free(dev->intr_buf);
     dev->intr_buf = NULL;
 
-    rshim_lock();
     rshim_deref(bd);
-    rshim_unlock();
   }
 
+  rshim_unlock();
   return rc;
 }
 
@@ -735,10 +728,12 @@ static void rshim_usb_disconnect(struct libusb_device *usb_dev)
   rshim_usb_t *dev;
 
   rshim_lock();
+
   bd = rshim_find_by_dev(usb_dev);
-  rshim_unlock();
-  if (!bd)
+  if (!bd) {
+    rshim_unlock();
     return;
+  }
 
   dev = container_of(bd, rshim_usb_t, bd);
 
@@ -797,7 +792,6 @@ static void rshim_usb_disconnect(struct libusb_device *usb_dev)
     dev->handle = NULL;
   }
 
-  rshim_lock();
   rshim_deref(bd);
   rshim_unlock();
 }