Reintroduce parallel exec 72658

README.md

@@ -150,7 +150,13 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and *
         // [SINGLE TESTCASE]: See below for more information
 
         // We also support the external loading of a complete test:
-        "@pathToTest.json"
+        "@pathToTest.json",
+
+        // By prefixing it with a number, the testtool runs that many instances of
+        // the included test file in parallel to each other.
+        //
+        // Only tests directly included by the manifest are allowed to run in parallel.
+        "5@pathToTestsThatShouldRunInParallel.json"
     ]
 }
 ```
@@ -160,8 +166,9 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and *
 ### manifest.json
 ```yaml
 {
-    // Define if the testuite should continue even if this test fails. (default:false)
+    // Define if the test suite should continue even if this test fails. (default: false)
     "continue_on_failure": true,
+
     // Name to identify this single test. Is important for the log. Try to give an explaning name
     "name": "Testname",
 
@@ -413,6 +420,31 @@ However that one will be stripped before parsing the template, which would be ju
 
 ** Unlike with delimiters, external tests/requests/responses don't inherit those removals, and need to be specified per file.
 
+## Run tests in parallel
+The tool is able to run tests in parallel to themselves. You activate this
+mechanism by including an external test file with `N@pathtofile.json`, where N
+is the number of parallel "clones" you want to have of the included tests.
+
+The included tests themselves are still run serially, only the entire set of
+tests will run in parallel for the specified number of replications.
+
+This is useful e.g. for stress-testing an API.
+
+Only tests directly included by a manifest are allowed to run in parallel.
+
+```yaml
+{
+    "name": "Binary Comparison",
+    "request": {
+        "endpoint": "suggest",
+        "method": "GET"
+    },
+
+    // Path to binary file with N@
+    "response": "123@simple.bin"
+}
+```
+
 ## Binary data comparison
 
 The tool is able to do a comparison with a binary file. Here we take a MD5 hash of the file and and then later compare

api_testsuite.go

@@ -9,6 +9,8 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/pkg/errors"
@@ -45,7 +47,7 @@ type Suite struct {
 	manifestPath    string
 	reporterRoot    *report.ReportElement
 	index           int
-	serverURL       string
+	serverURL       *url.URL
 	httpServer      http.Server
 	httpServerProxy *httpproxy.Proxy
 	httpServerDir   string
@@ -149,6 +151,12 @@ func NewTestSuite(config TestToolConfig, manifestPath string, manifestDir string
 	//Append suite manifest path to name, so we know in an automatic setup where the test is loaded from
 	suite.Name = fmt.Sprintf("%s (%s)", suite.Name, manifestPath)
 
+	// Parse serverURL
+	suite.serverURL, err = url.Parse(suite.Config.ServerURL)
+	if err != nil {
+		return nil, fmt.Errorf("can not load server url : %s", err)
+	}
+
 	// init store
 	err = suite.datastore.SetMap(suite.Store)
 	if err != nil {
@@ -179,8 +187,18 @@ func (ats *Suite) Run() bool {
 	success := true
 	for k, v := range ats.Tests {
 		child := r.NewChild(strconv.Itoa(k))
-		sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, child, ats.loader)
+
+		sTestSuccess := ats.parseAndRunTest(
+			v,
+			ats.manifestDir,
+			ats.manifestPath,
+			child,
+			ats.buildLoader(ats.loader, -1),
+			true, // parallel exec allowed for top-level tests
+		)
+
 		child.Leave(sTestSuccess)
+
 		if !sTestSuccess {
 			success = false
 			break
@@ -213,21 +231,43 @@ type TestContainer struct {
 	Path     string
 }
 
-func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *report.ReportElement, rootLoader template.Loader) bool {
-	//Init variables
-	// logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters)
+func (ats *Suite) buildLoader(rootLoader template.Loader, parallelRunIdx int) template.Loader {
 	loader := template.NewLoader(ats.datastore)
 	loader.Delimiters = rootLoader.Delimiters
 	loader.HTTPServerHost = ats.HTTPServerHost
-	serverURL, err := url.Parse(ats.Config.ServerURL)
-	if err != nil {
-		logrus.Error(fmt.Errorf("can not load server url into test (%s): %s", testFilePath, err))
-		return false
-	}
-	loader.ServerURL = serverURL
+	loader.ServerURL = ats.serverURL
 	loader.OAuthClient = ats.Config.OAuthClient
+	loader.ParallelRunIdx = parallelRunIdx
+
+	return loader
+}
 
-	//Get the Manifest with @ logic
+func (ats *Suite) parseAndRunTest(
+	v any, manifestDir, testFilePath string, r *report.ReportElement,
+	loader template.Loader, allowParallelExec bool,
+) bool {
+	// Parse PathSpec (if any) and determine number of parallel runs
+	parallelRuns := 1
+	if vStr, ok := v.(string); ok {
+		pathSpec, err := util.ParsePathSpec(vStr)
+		if err != nil {
+			logrus.Error(fmt.Errorf("test string is not a valid path spec: %w", err))
+			return false
+		}
+
+		parallelRuns = pathSpec.ParallelRuns
+	}
+
+	// Configure parallel runs, if specified
+	if parallelRuns > 1 {
+		// Check that parallel runs are actually allowed
+		if !allowParallelExec {
+			logrus.Error(fmt.Errorf("parallel runs are not allowed in nested tests (%s)", testFilePath))
+			return false
+		}
+	}
+
+	// Get the Manifest with @ logic
 	fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir)
 	dir := filepath.Dir(fileh)
 	if fileh != "" {
@@ -265,45 +305,69 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re
 	}
 
 	// Execute test cases
-	for i, testCase := range testCases {
-		var success bool
-
-		// If testCase can be unmarshalled as string, we may have a
-		// reference to another test using @ notation at hand
-		var testCaseStr string
-		err = util.Unmarshal(testCase, &testCaseStr)
-		if err == nil && util.IsPathSpec(testCaseStr) {
-			// Recurse if the testCase points to another file using @ notation
-			success = ats.parseAndRunTest(
-				testCaseStr,
-				filepath.Join(manifestDir, dir),
-				testFilePath,
-				r,
-				loader,
-			)
-		} else {
-			// Otherwise simply run the literal test case
-			success = ats.runLiteralTest(
-				TestContainer{
-					CaseByte: testCase,
-					Path:     filepath.Join(manifestDir, dir),
-				},
-				r,
-				testFilePath,
-				loader,
-				i,
-			)
-		}
-
-		if !success {
-			return false
-		}
-	}
-
-	return true
+	var successCount atomic.Uint32
+	var waitGroup sync.WaitGroup
+
+	waitGroup.Add(parallelRuns)
+
+	for runIdx := range parallelRuns {
+		runIdxCapture := runIdx
+
+		go func() {
+			defer waitGroup.Done()
+
+			for testIdx, testCase := range testCases {
+				var success bool
+
+				caseLoader := ats.buildLoader(loader, runIdxCapture)
+
+				// If testCase can be unmarshalled as string, we may have a
+				// reference to another test using @ notation at hand
+				var testCaseStr string
+				err = util.Unmarshal(testCase, &testCaseStr)
+				if err == nil && util.IsPathSpec(testCaseStr) {
+					// Recurse if the testCase points to another file using @ notation
+					success = ats.parseAndRunTest(
+						testCaseStr,
+						filepath.Join(manifestDir, dir),
+						testFilePath,
+						r,
+						caseLoader,
+						false, // no parallel exec allowed in nested tests
+					)
+				} else {
+					// Otherwise simply run the literal test case
+					success = ats.runLiteralTest(
+						TestContainer{
+							CaseByte: testCase,
+							Path:     filepath.Join(manifestDir, dir),
+						},
+						r,
+						testFilePath,
+						caseLoader,
+						runIdxCapture*len(testCases)+testIdx,
+					)
+				}
+
+				if !success {
+					// note that successCount is not incremented
+					return
+				}
+			}
+
+			successCount.Add(1)
+		}()
+	}
+
+	waitGroup.Wait()
+
+	return successCount.Load() == uint32(parallelRuns)
 }
 
-func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool {
+func (ats *Suite) runLiteralTest(
+	tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader,
+	index int,
+) bool {
 	r.SetName(testFilePath)
 
 	var test Case
@@ -320,7 +384,7 @@ func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, test
 	test.loader = loader
 	test.manifestDir = tc.Path
 	test.suiteIndex = ats.index
-	test.index = k
+	test.index = index
 	test.dataStore = ats.datastore
 	test.standardHeader = ats.StandardHeader
 	test.standardHeaderFromStore = ats.StandardHeaderFromStore

config.go

@@ -75,6 +75,9 @@ func NewTestToolConfig(serverURL string, rootDirectory []string, logNetwork bool
 		LogShort:       logShort,
 		OAuthClient:    Config.Apitest.OAuthClient,
 	}
+
+	config.fillInOAuthClientNames()
+
 	err = config.extractTestDirectories()
 	return config, err
 }
@@ -116,3 +119,14 @@ func (config *TestToolConfig) extractTestDirectories() error {
 	}
 	return nil
 }
+
+// fillInOAuthClientNames fills in the Client field of loaded OAuthClientConfig
+// structs, which the user may have left unset in the config yaml file.
+func (config *TestToolConfig) fillInOAuthClientNames() {
+	for key, clientConfig := range config.OAuthClient {
+		if clientConfig.Client == "" {
+			clientConfig.Client = key
+			config.OAuthClient[key] = clientConfig
+		}
+	}
+}

pkg/lib/datastore/datastore.go

@@ -16,15 +16,15 @@ type Datastore struct {
 	storage      map[string]any // custom storage
 	responseJson []string       // store the responses
 	logDatastore bool
-	lock         *sync.Mutex
+	lock         *sync.RWMutex
 }
 
 func NewStore(logDatastore bool) *Datastore {
 	ds := Datastore{}
 	ds.storage = make(map[string]any, 0)
 	ds.responseJson = make([]string, 0)
 	ds.logDatastore = logDatastore
-	ds.lock = &sync.Mutex{}
+	ds.lock = &sync.RWMutex{}
 	return &ds
 }
 
@@ -84,11 +84,17 @@ func (ds *Datastore) Delete(k string) {
 
 // We store the response
 func (ds *Datastore) UpdateLastResponse(s string) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
 	ds.responseJson[len(ds.responseJson)-1] = s
 }
 
 // We store the response
 func (ds *Datastore) AppendResponse(s string) {
+	ds.lock.Lock()
+	defer ds.lock.Unlock()
+
 	ds.responseJson = append(ds.responseJson, s)
 }
 
@@ -167,6 +173,9 @@ func (ds *Datastore) Set(index string, value any) error {
 }
 
 func (ds Datastore) Get(index string) (res any, err error) {
+	ds.lock.RLock()
+	defer ds.lock.RUnlock()
+
 	// strings are evalulated as int, so
 	// that we can support "-<int>" notations