switched from expect_fail to reverse_test_result, with some minor log…

…ic adjustment
programmfabrik · Jul 2, 2020 · 2767532 · 2767532
1 parent 2e27185
commit 2767532
Show file tree

Hide file tree

Showing 3 changed files with 11 additions and 9 deletions.
diff --git a/README.md b/README.md
@@ -288,8 +288,8 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and *
         "@continue_response_processing.json"
     ],
 
-    // If set to true, the test case will consider its failure as a success, useful for testing the test suite
-    "expect_fail": false
+    // If set to true, the test case will consider its failure as a success, and the other way around
+    "reverse_test_result": false
 }
 ```
 

diff --git a/api_testcase.go b/api_testcase.go
@@ -52,7 +52,7 @@ type Case struct {
 	standardHeaderFromStore map[string]string
 
 	ServerURL string `json:"server_url"`
-	ExpectToFail bool `json:"expect_fail"`
+	ReverseTestResult bool `json:"reverse_test_result"`
 }
 
 func (testCase Case) runAPITestCase(parentReportElem *report.ReportElement) bool {
@@ -99,8 +99,10 @@ func (testCase Case) runAPITestCase(parentReportElem *report.ReportElement) bool
 		success = false
 	}
 
-	// XOR for when we want it to fail
-	success = success != testCase.ExpectToFail
+	// Reverse if needed
+	if testCase.ReverseTestResult {
+		success = !success
+	}
 
 	if !success {
 		logrus.WithFields(logrus.Fields{"elapsed": elapsed.Seconds()}).Warnf("     [%2d] failure", testCase.index)
@@ -289,7 +291,7 @@ func (testCase Case) executeRequest(counter int) (compare.CompareResult, api.Req
 func (testCase Case) LogResp(response api.Response) {
 	errString := fmt.Sprintf("[RESPONSE]:\n%s\n\n", limitLines(response.ToString(), Config.Apitest.Limit.Response))
 
-	if !testCase.ExpectToFail && testCase.LogNetwork != nil && !*testCase.LogNetwork && !testCase.ContinueOnFailure {
+	if !testCase.ReverseTestResult && testCase.LogNetwork != nil && !*testCase.LogNetwork && !testCase.ContinueOnFailure {
 		testCase.ReportElem.SaveToReportLogF(errString)
 		logrus.Debug(errString)
 	}
@@ -299,7 +301,7 @@ func (testCase Case) LogResp(response api.Response) {
 func (testCase Case) LogReq(req api.Request) {
 	errString := fmt.Sprintf("[REQUEST]:\n%s\n\n", limitLines(req.ToString(logCurl), Config.Apitest.Limit.Request))
 
-	if !testCase.ExpectToFail && !testCase.ContinueOnFailure && testCase.LogNetwork != nil && *testCase.LogNetwork == false {
+	if !testCase.ReverseTestResult && !testCase.ContinueOnFailure && testCase.LogNetwork != nil && *testCase.LogNetwork == false {
 		testCase.ReportElem.SaveToReportLogF(errString)
 		logrus.Debug(errString)
 	}
@@ -400,7 +402,7 @@ func (testCase Case) run() (bool, error) {
 	}
 
 	if !responsesMatch.Equal || timedOutFlag {
-		if !testCase.ExpectToFail {
+		if !testCase.ReverseTestResult {
 			for _, v := range responsesMatch.Failures {
 				logrus.Errorf("[%s] %s", v.Key, v.Message)
 				r.SaveToReportLog(fmt.Sprintf("[%s] %s", v.Key, v.Message))

diff --git a/test/control/manifest.json b/test/control/manifest.json
@@ -136,7 +136,7 @@
                     }
                 }
             },
-            "expect_fail": true
+            "reverse_test_result": true
         }
     ]
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -136,7 +136,7 @@ @@
                         }
                     }
                 },
-                "expect_fail": true
+                "reverse_test_result": true
             }
         ]
     }