Merge pull request #450 from mq-cloudpak/amf-miqm-failover-backport-930

Allow for slow standby take-over in MIQM test (backport)

test/docker/mq_multi_instance_test.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019, 2020
+© Copyright IBM Corporation 2019, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"strings"
 	"testing"
 	"time"
@@ -92,15 +93,28 @@ func TestMultiInstanceContainerStop(t *testing.T) {
 	waitForReady(t, cli, qm1aId)
 	waitForReady(t, cli, qm1bId)
 
-	err, active, standby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
+	err, originalActive, originalStandby := getActiveStandbyQueueManager(t, cli, qm1aId, qm1bId)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	stopContainer(t, cli, active)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	stopContainer(t, cli, originalActive)
 
-	if status := getQueueManagerStatus(t, cli, standby, "QM1"); strings.Compare(status, "Running") != 0 {
-		t.Fatalf("Expected QM1 to be running as active queue manager, dspmq returned status of %v", status)
+	for {
+		status := getQueueManagerStatus(t, cli, originalStandby, "QM1")
+		select {
+		case <-time.After(1 * time.Second):
+			if status == "Running" {
+				t.Logf("Original standby is now the active")
+				return
+			} else if status == "Starting" {
+				t.Logf("Original standby is starting")
+			}
+		case <-ctx.Done():
+			t.Fatalf("%s Timed out waiting for standby to become the active.  Status=%v", time.Now().Format(time.RFC3339), status)
+		}
 	}
 }
 

test/docker/mq_multi_instance_test_util.go

@@ -1,5 +1,5 @@
 /*
-© Copyright IBM Corporation 2019
+© Copyright IBM Corporation 2019, 2023
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -77,6 +77,7 @@ func getActiveStandbyQueueManager(t *testing.T, cli *client.Client, qm1aId strin
 
 func getQueueManagerStatus(t *testing.T, cli *client.Client, containerID string, queueManagerName string) string {
 	_, dspmqOut := execContainer(t, cli, containerID, "", []string{"bash", "-c", "dspmq", "-m", queueManagerName})
+	t.Logf("dspmq for %v (%v) returned: %v", containerID, queueManagerName, dspmqOut)
 	regex := regexp.MustCompile(`STATUS\(.*\)`)
 	status := regex.FindString(dspmqOut)
 	status = strings.TrimSuffix(strings.TrimPrefix(status, "STATUS("), ")")