Keep retry registering node controller with RegistryServer

Instead of exiting with error, driver has to wait and retry 'node controller'
registration till the registry server up and registration get succeed.

FIXES: intel#182

Signed-off-by: Amarnath Valluri <amarnath.valluri@intel.com>

pkg/pmem-csi-driver/pmem-csi-driver.go

@@ -17,6 +17,8 @@ import (
 	pmemgrpc "github.com/intel/pmem-csi/pkg/pmem-grpc"
 	registry "github.com/intel/pmem-csi/pkg/pmem-registry"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	"k8s.io/klog/glog"
 )
 
@@ -194,31 +196,40 @@ func (pmemd *pmemDriver) Run() error {
 }
 
 func (pmemd *pmemDriver) registerNodeController() error {
-	fmt.Printf("Connecting to Registry at : %s\n", pmemd.cfg.RegistryEndpoint)
 	var err error
 	var conn *grpc.ClientConn
 
+	req := registry.RegisterControllerRequest{
+		NodeId:   pmemd.cfg.NodeID,
+		Endpoint: pmemd.cfg.ControllerEndpoint,
+	}
+
 	for {
+		glog.Infof("Connecting to registry server at: %s\n", pmemd.cfg.RegistryEndpoint)
 		conn, err = pmemgrpc.Connect(pmemd.cfg.RegistryEndpoint, pmemd.clientTLSConfig, connectionTimeout)
 		if err == nil {
-			glog.Info("Connected to RegistryServer!!!")
 			break
 		}
-		/* TODO: Retry loop */
-		glog.Infof("Failed to connect RegistryServer : %s, retrying after 10 seconds...", err.Error())
-		time.Sleep(10 * time.Second)
+		glog.Infof("Failed to connect registry server: %s, retrying after %v seconds...", err.Error(), retryTimeout.Seconds())
+		time.Sleep(retryTimeout)
 	}
-	client := registry.NewRegistryClient(conn)
-	req := registry.RegisterControllerRequest{
-		NodeId:   pmemd.cfg.NodeID,
-		Endpoint: pmemd.cfg.ControllerEndpoint,
-	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
-	defer cancel()
-
-	if _, err = client.RegisterController(ctx, &req); err != nil {
-		return fmt.Errorf("Fail to register with server at '%s' : %s", pmemd.cfg.RegistryEndpoint, err.Error())
+	for {
+		client := registry.NewRegistryClient(conn)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+
+		glog.Info("Registering controller...")
+		if _, err = client.RegisterController(ctx, &req); err != nil {
+			if s, ok := status.FromError(err); ok && s.Code() == codes.InvalidArgument {
+				return fmt.Errorf("Registration filed: %s", s.Message())
+			}
+			glog.Infof("Failed to register: %s, retrying after %v seconds...", err.Error(), retryTimeout.Seconds())
+			time.Sleep(retryTimeout)
+		} else {
+			break
+		}
 	}
+	glog.Info("Registration success!!!")
 
 	return nil
 }