Re-introducing Parallel Execution, in a simplified version

Parallel Execution is re-introduced, but in a much simplified version.
p@foo.json / pN@foo.json notation is gone, replaced by N@foo.json.

When including a file with N@foo.json, its tests are still run serially,
but the entire file is run in N parallel goroutines.

README.md

@@ -150,7 +150,14 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and *
         // [SINGLE TESTCASE]: See below for more information
 
         // We also support the external loading of a complete test:
-        "@pathToTest.json"
+        "@pathToTest.json",
+
+        // By prefixing it with a number, the testtool runs that many instances of
+        // the included test file in parallel to each other.
+        //
+        // Only tests directly included by the manifest are allowed to run in parallel.
+        // All parallel tests are set to ContinueOnFailure.
+        "5@pathToTestsThatShouldRunInParallel.json"
     ]
 }
 ```
@@ -413,6 +420,34 @@ However that one will be stripped before parsing the template, which would be ju
 
 ** Unlike with delimiters, external tests/requests/responses don't inherit those removals, and need to be specified per file.
 
+## Run tests in parallel
+The tool is able to run tests in parallel to themselves. You activate this
+mechanism by including an external test file with `N@pathtofile.json`, where N
+is the number of parallel "clones" you want to have of the included tests.
+
+The included tests themselves are still run serially, only the entire set of
+tests will run in parallel for the specified number of replications.
+
+This is useful e.g. for stress-testing an API.
+
+Only tests directly included by a manifest are allowed to run in parallel.
+
+**All tests that are run in parallel are implicitly set to ContinueOnFailure,
+since otherwise the log/report output would be confusing.**
+
+```yaml
+{
+    "name": "Binary Comparison",
+    "request": {
+        "endpoint": "suggest",
+        "method": "GET"
+    },
+
+    // Path to binary file with N@
+    "response": "123@simple.bin"
+}
+```
+
 ## Binary data comparison
 
 The tool is able to do a comparison with a binary file. Here we take a MD5 hash of the file and and then later compare

api_testsuite.go

@@ -179,8 +179,19 @@ func (ats *Suite) Run() bool {
 	success := true
 	for k, v := range ats.Tests {
 		child := r.NewChild(strconv.Itoa(k))
-		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, child, ats.loader)
+
+		sTestSuccess := ats.parseAndRunTest(
+			v,
+			ats.manifestDir,
+			ats.manifestPath,
+			child,
+			ats.loader,
+			true,  // parallel exec allowed for top-level tests
+			false, // don't force ContinueOnFail at this point
+		)
+
 		child.Leave(sTestSuccess)
+
 		if !sTestSuccess {
 			success = false
 			break
@@ -213,7 +224,10 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *report.ReportElement, rootLoader template.Loader) bool {
+func (ats *Suite) parseAndRunTest(
+	v any, manifestDir, testFilePath string, r *report.ReportElement,
+	rootLoader template.Loader, allowParallelExec bool, forceContinueOnFail bool,
+) bool {
 	//Init variables
 	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
 	loader := template.NewLoader(ats.datastore)
@@ -227,7 +241,28 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re
 	loader.ServerURL = serverURL
 	loader.OAuthClient = ats.Config.OAuthClient
 
-	//Get the Manifest with @ logic
+	// Determine number of parallel repetitions, if any
+	repetitions := 1
+	if vStr, ok := v.(string); ok {
+		pathSpec, ok := util.ParsePathSpec(vStr)
+		if ok {
+			repetitions = pathSpec.Repetitions
+		}
+	}
+
+	// Configure parallel repetitions, if specified
+	if repetitions > 1 {
+		// Check that parallel repetitions are actually allowed
+		if !allowParallelExec {
+			logrus.Error(fmt.Errorf("parallel repetitions are not allowed in nested tests (%s)", testFilePath))
+			return false
+		}
+
+		// If we're running in parallel repetition mode, force subordinate tests to ContinueOnFail
+		forceContinueOnFail = true
+	}
+
+	// Get the Manifest with @ logic
 	fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir)
 	dir := filepath.Dir(fileh)
 	if fileh != "" {
@@ -265,45 +300,66 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re
 	}
 
 	// Execute test cases
-	for i, testCase := range testCases {
-		var success bool
-
-		// If testCase can be unmarshalled as string, we may have a
-		// reference to another test using @ notation at hand
-		var testCaseStr string
-		err = util.Unmarshal(testCase, &testCaseStr)
-		if err == nil && util.IsPathSpec(testCaseStr) {
-			// Recurse if the testCase points to another file using @ notation
-			success = ats.parseAndRunTest(
-				testCaseStr,
-				filepath.Join(manifestDir, dir),
-				testFilePath,
-				r,
-				loader,
-			)
-		} else {
-			// Otherwise simply run the literal test case
-			success = ats.runLiteralTest(
-				TestContainer{
-					CaseByte: testCase,
-					Path:     filepath.Join(manifestDir, dir),
-				},
-				r,
-				testFilePath,
-				loader,
-				i,
-			)
-		}
-
-		if !success {
+	successCh := make(chan bool, repetitions)
+
+	for repeatIdx := range repetitions {
+		go func() {
+			for testIdx, testCase := range testCases {
+				var success bool
+
+				// If testCase can be unmarshalled as string, we may have a
+				// reference to another test using @ notation at hand
+				var testCaseStr string
+				err = util.Unmarshal(testCase, &testCaseStr)
+				if err == nil && util.IsPathSpec(testCaseStr) {
+					// Recurse if the testCase points to another file using @ notation
+					success = ats.parseAndRunTest(
+						testCaseStr,
+						filepath.Join(manifestDir, dir),
+						testFilePath,
+						r,
+						loader,
+						false, // no parallel exec allowed in nested tests
+						forceContinueOnFail,
+					)
+				} else {
+					// Otherwise simply run the literal test case
+					success = ats.runLiteralTest(
+						TestContainer{
+							CaseByte: testCase,
+							Path:     filepath.Join(manifestDir, dir),
+						},
+						r,
+						testFilePath,
+						loader,
+						repeatIdx*len(testCases)+testIdx,
+						forceContinueOnFail,
+					)
+				}
+
+				if !success {
+					successCh <- false
+					return
+				}
+			}
+
+			successCh <- true
+		}()
+	}
+
+	for range repetitions {
+		if success := <-successCh; !success {
 			return false
 		}
 	}
 
 	return true
 }
 
-func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool {
+func (ats *Suite) runLiteralTest(
+	tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader,
+	index int, forceContinueOnFailure bool,
+) bool {
 	r.SetName(testFilePath)
 
 	var test Case
@@ -320,10 +376,13 @@ func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, test
 	test.loader = loader
 	test.manifestDir = tc.Path
 	test.suiteIndex = ats.index
-	test.index = k
+	test.index = index
 	test.dataStore = ats.datastore
 	test.standardHeader = ats.StandardHeader
 	test.standardHeaderFromStore = ats.StandardHeaderFromStore
+	if forceContinueOnFailure {
+		test.ContinueOnFailure = true
+	}
 	if test.LogNetwork == nil {
 		test.LogNetwork = &ats.Config.LogNetwork
 	}