diff --git a/README.md b/README.md index f083d4a..0ce9e7e 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,14 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * // [SINGLE TESTCASE]: See below for more information // We also support the external loading of a complete test: - "@pathToTest.json" + "@pathToTest.json", + + // By prefixing it with a number, the testtool runs that many instances of + // the included test file in parallel to each other. + // + // Only tests directly included by the manifest are allowed to run in parallel. + // All parallel tests are set to ContinueOnFailure. + "5@pathToTestsThatShouldRunInParallel.json" ] } ``` @@ -413,6 +420,34 @@ However that one will be stripped before parsing the template, which would be ju ** Unlike with delimiters, external tests/requests/responses don't inherit those removals, and need to be specified per file. +## Run tests in parallel +The tool is able to run tests in parallel to themselves. You activate this +mechanism by including an external test file with `N@pathtofile.json`, where N +is the number of parallel "clones" you want to have of the included tests. + +The included tests themselves are still run serially, only the entire set of +tests will run in parallel for the specified number of replications. + +This is useful e.g. for stress-testing an API. + +Only tests directly included by a manifest are allowed to run in parallel. + +**All tests that are run in parallel are implicitly set to ContinueOnFailure, +since otherwise the log/report output would be confusing.** + +```yaml +{ + "name": "Binary Comparison", + "request": { + "endpoint": "suggest", + "method": "GET" + }, + + // Path to binary file with N@ + "response": "123@simple.bin" +} +``` + ## Binary data comparison The tool is able to do a comparison with a binary file. Here we take a MD5 hash of the file and and then later compare diff --git a/api_testsuite.go b/api_testsuite.go index 6d9cbae..7f8f311 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -179,8 +179,19 @@ func (ats *Suite) Run() bool { success := true for k, v := range ats.Tests { child := r.NewChild(strconv.Itoa(k)) - sTestSuccess := ats.parseAndRunTest(v, ats.manifestDir, ats.manifestPath, child, ats.loader) + + sTestSuccess := ats.parseAndRunTest( + v, + ats.manifestDir, + ats.manifestPath, + child, + ats.loader, + true, // parallel exec allowed for top-level tests + false, // don't force ContinueOnFail at this point + ) + child.Leave(sTestSuccess) + if !sTestSuccess { success = false break @@ -213,7 +224,10 @@ type TestContainer struct { Path string } -func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *report.ReportElement, rootLoader template.Loader) bool { +func (ats *Suite) parseAndRunTest( + v any, manifestDir, testFilePath string, r *report.ReportElement, + rootLoader template.Loader, allowParallelExec bool, forceContinueOnFail bool, +) bool { //Init variables // logrus.Warnf("Test %s, Prev delimiters: %#v", testFilePath, rootLoader.Delimiters) loader := template.NewLoader(ats.datastore) @@ -227,7 +241,28 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re loader.ServerURL = serverURL loader.OAuthClient = ats.Config.OAuthClient - //Get the Manifest with @ logic + // Determine number of parallel repetitions, if any + repetitions := 1 + if vStr, ok := v.(string); ok { + pathSpec, ok := util.ParsePathSpec(vStr) + if ok { + repetitions = pathSpec.Repetitions + } + } + + // Configure parallel repetitions, if specified + if repetitions > 1 { + // Check that parallel repetitions are actually allowed + if !allowParallelExec { + logrus.Error(fmt.Errorf("parallel repetitions are not allowed in nested tests (%s)", testFilePath)) + return false + } + + // If we're running in parallel repetition mode, force subordinate tests to ContinueOnFail + forceContinueOnFail = true + } + + // Get the Manifest with @ logic fileh, testObj, err := template.LoadManifestDataAsRawJson(v, manifestDir) dir := filepath.Dir(fileh) if fileh != "" { @@ -265,37 +300,55 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re } // Execute test cases - for i, testCase := range testCases { - var success bool - - // If testCase can be unmarshalled as string, we may have a - // reference to another test using @ notation at hand - var testCaseStr string - err = util.Unmarshal(testCase, &testCaseStr) - if err == nil && util.IsPathSpec(testCaseStr) { - // Recurse if the testCase points to another file using @ notation - success = ats.parseAndRunTest( - testCaseStr, - filepath.Join(manifestDir, dir), - testFilePath, - r, - loader, - ) - } else { - // Otherwise simply run the literal test case - success = ats.runLiteralTest( - TestContainer{ - CaseByte: testCase, - Path: filepath.Join(manifestDir, dir), - }, - r, - testFilePath, - loader, - i, - ) - } - - if !success { + successCh := make(chan bool, repetitions) + + for repeatIdx := range repetitions { + go func() { + for testIdx, testCase := range testCases { + var success bool + + // If testCase can be unmarshalled as string, we may have a + // reference to another test using @ notation at hand + var testCaseStr string + err = util.Unmarshal(testCase, &testCaseStr) + if err == nil && util.IsPathSpec(testCaseStr) { + // Recurse if the testCase points to another file using @ notation + success = ats.parseAndRunTest( + testCaseStr, + filepath.Join(manifestDir, dir), + testFilePath, + r, + loader, + false, // no parallel exec allowed in nested tests + forceContinueOnFail, + ) + } else { + // Otherwise simply run the literal test case + success = ats.runLiteralTest( + TestContainer{ + CaseByte: testCase, + Path: filepath.Join(manifestDir, dir), + }, + r, + testFilePath, + loader, + repeatIdx*len(testCases)+testIdx, + forceContinueOnFail, + ) + } + + if !success { + successCh <- false + return + } + } + + successCh <- true + }() + } + + for range repetitions { + if success := <-successCh; !success { return false } } @@ -303,7 +356,10 @@ func (ats *Suite) parseAndRunTest(v any, manifestDir, testFilePath string, r *re return true } -func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, k int) bool { +func (ats *Suite) runLiteralTest( + tc TestContainer, r *report.ReportElement, testFilePath string, loader template.Loader, + index int, forceContinueOnFailure bool, +) bool { r.SetName(testFilePath) var test Case @@ -320,10 +376,13 @@ func (ats *Suite) runLiteralTest(tc TestContainer, r *report.ReportElement, test test.loader = loader test.manifestDir = tc.Path test.suiteIndex = ats.index - test.index = k + test.index = index test.dataStore = ats.datastore test.standardHeader = ats.StandardHeader test.standardHeaderFromStore = ats.StandardHeaderFromStore + if forceContinueOnFailure { + test.ContinueOnFailure = true + } if test.LogNetwork == nil { test.LogNetwork = &ats.Config.LogNetwork }