Re-introducing Parallel Execution

Note that Parallel Execution is only available for top-level tests.

api_testsuite.go

@@ -179,7 +179,15 @@ func (ats *Suite) Run() bool {
 	success := true
 	for k, v := range ats.Tests {
 		child := r.NewChild(strconv.Itoa(k))
-		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, child, ats.loader)
+		sTestSuccess := ats.parseAndRunTest(
+			v,
+			ats.manifestDir,
+			ats.manifestPath,
+			child,
+			ats.loader,
+			true,  // parallel exec allowed for top-level tests
+			false, // don't force ContinueOnFail at this point
+		)
 		child.Leave(sTestSuccess)
 		if !sTestSuccess {
 			success = false
@@ -213,7 +221,10 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *report.ReportElement, rootLoader template.Loader) bool {
+func (ats *Suite) parseAndRunTest(
+	v any, manifestDir, testFilePath string, r *report.ReportElement,
+	rootLoader template.Loader, allowParallelExec bool, forceContinueOnFail bool,
+) bool {
 	//Init variables
 	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
 	loader := template.NewLoader(ats.datastore)
@@ -227,6 +238,19 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re
 	loader.ServerURL = serverURL
 	loader.OAuthClient = ats.Config.OAuthClient
 
+	// Determine number of parallel repetitions
+	parallelRepetitions := 1
+	if vStr, ok := v.(string); ok && allowParallelExec {
+		if util.IsParallelPathSpec(vStr) {
+			parallelRepetitions, _ = util.GetParallelPathSpec(vStr)
+		}
+	}
+
+	// If we're running in parallel repetition mode, force subordinate tests to ContinueOnFail
+	if allowParallelExec && parallelRepetitions > 1 {
+		forceContinueOnFail = true
+	}
+
 	//Get the Manifest with @ logic
 	fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir)
 	dir := filepath.Dir(fileh)
@@ -266,44 +290,62 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re
 
 	// Execute test cases
 	for i, testCase := range testCases {
-		var success bool
-
-		// If testCase can be unmarshalled as string, we may have a
-		// reference to another test using @ notation at hand
-		var testCaseStr string
-		err = util.Unmarshal(testCase, &testCaseStr)
-		if err == nil && util.IsPathSpec(testCaseStr) {
-			// Recurse if the testCase points to another file using @ notation
-			success = ats.parseAndRunTest(
-				testCaseStr,
-				filepath.Join(manifestDir, dir),
-				testFilePath,
-				r,
-				loader,
-			)
-		} else {
-			// Otherwise simply run the literal test case
-			success = ats.runLiteralTest(
-				TestContainer{
-					CaseByte: testCase,
-					Path:     filepath.Join(manifestDir, dir),
-				},
-				r,
-				testFilePath,
-				loader,
-				i,
-			)
+		successCh := make(chan bool, parallelRepetitions)
+
+		for j := range parallelRepetitions {
+			go func() {
+				// If testCase can be unmarshalled as string, we may have a
+				// reference to another test using @ notation at hand
+				var testCaseStr string
+				err = util.Unmarshal(testCase, &testCaseStr)
+				if err == nil && util.IsPathSpec(testCaseStr) {
+					// Recurse if the testCase points to another file using @ notation
+					successCh <- ats.parseAndRunTest(
+						testCaseStr,
+						filepath.Join(manifestDir, dir),
+						testFilePath,
+						r,
+						loader,
+						false, // no parallel exec allowed in nested tests
+						forceContinueOnFail,
+					)
+				} else {
+					// Otherwise simply run the literal test case
+					successCh <- ats.runLiteralTest(
+						TestContainer{
+							CaseByte: testCase,
+							Path:     filepath.Join(manifestDir, dir),
+						},
+						r,
+						testFilePath,
+						loader,
+						i*parallelRepetitions+j, // FIXME: This index is not unique within the suite - this may become confusing in the output when tests run in parallel.
+						forceContinueOnFail,
+					)
+				}
+			}()
 		}
 
-		if !success {
+		// Wait for all goroutines, check for success
+		overallSuccess := true
+		for range parallelRepetitions {
+			success := <-successCh
+			if !success {
+				overallSuccess = false
+			}
+		}
+		if !overallSuccess {
 			return false
 		}
 	}
 
 	return true
 }
 
-func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool {
+func (ats *Suite) runLiteralTest(
+	tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader,
+	index int, forceContinueOnFailure bool,
+) bool {
 	r.SetName(testFilePath)
 
 	var test Case
@@ -320,10 +362,13 @@ func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, test
 	test.loader = loader
 	test.manifestDir = tc.Path
 	test.suiteIndex = ats.index
-	test.index = k
+	test.index = index
 	test.dataStore = ats.datastore
 	test.standardHeader = ats.StandardHeader
 	test.standardHeaderFromStore = ats.StandardHeaderFromStore
+	if forceContinueOnFailure {
+		test.ContinueOnFailure = true
+	}
 	if test.LogNetwork == nil {
 		test.LogNetwork = &ats.Config.LogNetwork
 	}