From d341bfd7214b6fc0e381d650dc3fc4fe0bf5c91c Mon Sep 17 00:00:00 2001 From: Philipp Hempel Date: Mon, 28 Oct 2024 12:17:10 +0100 Subject: [PATCH 1/6] - new response field header_flat: - map of strings with (concatenated) header values - contains the same values as the header, but not in arrays - :control can be used on different header values - removed unused body:control and header:control from code, check was never implemented see #73980 --- api_testcase.go | 5 +- api_testsuite.go | 2 + pkg/lib/api/request.go | 2 +- pkg/lib/api/response.go | 84 +++++++++++------------- pkg/lib/api/response_test.go | 6 +- pkg/lib/template/template_loader_test.go | 3 +- test/control/header/manifest.json | 15 ----- test/datastore/check.json | 6 +- test/response/header/manifest.json | 84 ++++++++++++++++++++++++ 9 files changed, 135 insertions(+), 72 deletions(-) create mode 100644 test/response/header/manifest.json diff --git a/api_testcase.go b/api_testcase.go index 4532e22d..42bdd6ce 100644 --- a/api_testcase.go +++ b/api_testcase.go @@ -50,6 +50,7 @@ type Case struct { dataStore *datastore.Datastore standardHeader map[string]*string + headerFlat map[string]*string standardHeaderFromStore map[string]string ServerURL string `json:"server_url"` @@ -471,7 +472,7 @@ func (testCase Case) loadRequest() (api.Request, error) { func (testCase Case) loadExpectedResponse() (res api.Response, err error) { // unspecified response is interpreted as status_code 200 if testCase.ResponseData == nil { - return api.NewResponse(http.StatusOK, nil, nil, nil, nil, nil, res.Format) + return api.NewResponse(http.StatusOK, nil, nil, nil, nil, res.Format) } spec, err := testCase.loadResponseSerialization(testCase.ResponseData) if err != nil { @@ -489,7 +490,7 @@ func (testCase Case) responsesEqual(expected, got api.Response) (compare.Compare if err != nil { return compare.CompareResult{}, fmt.Errorf("error loading expected generic json: %s", err) } - if len(expected.Body) == 0 && len(expected.BodyControl) == 0 { + if len(expected.Body) == 0 { expected.Format.IgnoreBody = true } else { expected.Format.IgnoreBody = false diff --git a/api_testsuite.go b/api_testsuite.go index e3c2582d..fc4b5f3c 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -44,6 +44,7 @@ type Suite struct { Store map[string]any `json:"store"` StandardHeader map[string]*string `yaml:"header" json:"header"` + HeaderFlat map[string]*string `yaml:"header_flat" json:"header_flat"` StandardHeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` Config TestToolConfig @@ -399,6 +400,7 @@ func (ats *Suite) runLiteralTest( test.index = index test.dataStore = ats.datastore test.standardHeader = ats.StandardHeader + test.headerFlat = ats.HeaderFlat test.standardHeaderFromStore = ats.StandardHeaderFromStore if test.LogNetwork == nil { test.LogNetwork = &ats.Config.LogNetwork diff --git a/pkg/lib/api/request.go b/pkg/lib/api/request.go index 5d3e1d8a..a3a4619c 100755 --- a/pkg/lib/api/request.go +++ b/pkg/lib/api/request.go @@ -347,7 +347,7 @@ func (request Request) Send() (response Response, err error) { if err != nil { return response, err } - response, err = NewResponse(httpResponse.StatusCode, header, nil, httpResponse.Cookies(), httpResponse.Body, nil, ResponseFormat{}) + response, err = NewResponse(httpResponse.StatusCode, header, nil, httpResponse.Cookies(), httpResponse.Body, ResponseFormat{}) if err != nil { return response, fmt.Errorf("error constructing response from http response") } diff --git a/pkg/lib/api/response.go b/pkg/lib/api/response.go index 3f69a960..1d095c26 100755 --- a/pkg/lib/api/response.go +++ b/pkg/lib/api/response.go @@ -21,32 +21,28 @@ import ( ) type Response struct { - StatusCode int - Headers map[string]any - headerControl util.JsonObject - Cookies []*http.Cookie - Body []byte - BodyControl util.JsonObject - Format ResponseFormat + StatusCode int + Headers map[string]any + HeaderFlat map[string]any + Cookies []*http.Cookie + Body []byte + Format ResponseFormat ReqDur time.Duration BodyLoadDur time.Duration } -func (res Response) NeedsCheck() bool { - if res.StatusCode != http.StatusOK { - return true - } - if len(res.Headers) > 0 || len(res.Cookies) > 0 || len(res.Body) > 0 || len(res.BodyControl) > 0 { - return true - } - return false -} - -func (res Response) SerializeHeaders() (headers map[string]any, err error) { +func (res Response) SerializeHeaderFlat() (headers map[string]any, err error) { headers = map[string]any{} for k, h := range res.Headers { - headers[k] = h + switch v := h.(type) { + case string: + headers[k] = h + case nil: + headers[k] = "" + case []string: + headers[k] = strings.Join(v, "; ") + } } return headers, nil } @@ -73,13 +69,12 @@ type Cookie struct { } type ResponseSerialization struct { - StatusCode int `yaml:"statuscode" json:"statuscode"` - Headers map[string]any `yaml:"header" json:"header,omitempty"` - HeaderControl util.JsonObject `yaml:"header:control" json:"header:control,omitempty"` - Cookies map[string]Cookie `yaml:"cookie" json:"cookie,omitempty"` - Body any `yaml:"body" json:"body,omitempty"` - BodyControl util.JsonObject `yaml:"body:control" json:"body:control,omitempty"` - Format ResponseFormat `yaml:"format" json:"format,omitempty"` + StatusCode int `yaml:"statuscode" json:"statuscode"` + Headers map[string]any `yaml:"header" json:"header,omitempty"` + HeaderFlat map[string]any `yaml:"header_flat" json:"header_flat,omitempty"` + Cookies map[string]Cookie `yaml:"cookie" json:"cookie,omitempty"` + Body any `yaml:"body" json:"body,omitempty"` + Format ResponseFormat `yaml:"format" json:"format,omitempty"` } type ResponseFormat struct { @@ -91,14 +86,19 @@ type ResponseFormat struct { PreProcess *PreProcess `json:"pre_process,omitempty"` } -func NewResponse(statusCode int, headers map[string]any, headerControl util.JsonObject, cookies []*http.Cookie, body io.Reader, bodyControl util.JsonObject, bodyFormat ResponseFormat) (res Response, err error) { +func NewResponse(statusCode int, + headers map[string]any, + headerFlat map[string]any, + cookies []*http.Cookie, + body io.Reader, + bodyFormat ResponseFormat, +) (res Response, err error) { res = Response{ - StatusCode: statusCode, - Headers: headers, - Cookies: cookies, - BodyControl: bodyControl, - headerControl: headerControl, - Format: bodyFormat, + StatusCode: statusCode, + Headers: headers, + HeaderFlat: headerFlat, + Cookies: cookies, + Format: bodyFormat, } if body != nil { start := time.Now() @@ -144,7 +144,7 @@ func NewResponseFromSpec(spec ResponseSerialization) (res Response, err error) { } } - return NewResponse(spec.StatusCode, spec.Headers, spec.HeaderControl, cookies, body, spec.BodyControl, spec.Format) + return NewResponse(spec.StatusCode, spec.Headers, spec.HeaderFlat, cookies, body, spec.Format) } // ServerResponseToGenericJSON parse response from server. convert xml, csv, binary to json if necessary @@ -214,13 +214,14 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm return res, fmt.Errorf("Invalid response format '%s'", responseFormat.Type) } - headers, err := resp.SerializeHeaders() + headerFlat, err := resp.SerializeHeaderFlat() if err != nil { return res, err } responseJSON := ResponseSerialization{ StatusCode: resp.StatusCode, - Headers: headers, + Headers: resp.Headers, + HeaderFlat: headerFlat, } // Build cookies map from standard bag if len(resp.Cookies) > 0 { @@ -284,15 +285,10 @@ func (response Response) ToGenericJSON() (any, error) { } } - headers, err := response.SerializeHeaders() - if err != nil { - return res, err - } responseJSON := ResponseSerialization{ - StatusCode: response.StatusCode, - BodyControl: response.BodyControl, - Headers: headers, - HeaderControl: response.headerControl, + StatusCode: response.StatusCode, + Headers: response.Headers, + HeaderFlat: response.HeaderFlat, } // Build cookies map from standard bag diff --git a/pkg/lib/api/response_test.go b/pkg/lib/api/response_test.go index 4c0fd074..8767e674 100644 --- a/pkg/lib/api/response_test.go +++ b/pkg/lib/api/response_test.go @@ -73,7 +73,7 @@ func TestResponse_NewResponseFromSpec_StatusCode_not_set(t *testing.T) { } func TestResponse_NewResponse(t *testing.T) { - response, err := NewResponse(200, nil, nil, nil, strings.NewReader("foo"), nil, ResponseFormat{}) + response, err := NewResponse(200, nil, nil, nil, strings.NewReader("foo"), ResponseFormat{}) go_test_utils.ExpectNoError(t, err, "unexpected error") go_test_utils.AssertIntEquals(t, response.StatusCode, 200) } @@ -86,7 +86,7 @@ func TestResponse_String(t *testing.T) { } }` - response, err := NewResponse(200, nil, nil, nil, strings.NewReader(requestString), nil, ResponseFormat{}) + response, err := NewResponse(200, nil, nil, nil, strings.NewReader(requestString), ResponseFormat{}) go_test_utils.ExpectNoError(t, err, "error constructing response") assertString := "200\n\n\n" + requestString @@ -119,7 +119,7 @@ func TestResponse_Cookies(t *testing.T) { if err != nil { t.Fatal(err) } - response, err := NewResponse(res.StatusCode, header, nil, res.Cookies(), res.Body, nil, ResponseFormat{}) + response, err := NewResponse(res.StatusCode, header, nil, res.Cookies(), res.Body, ResponseFormat{}) if err != nil { t.Fatal(err) } diff --git a/pkg/lib/template/template_loader_test.go b/pkg/lib/template/template_loader_test.go index c3b839f7..485cb9b6 100644 --- a/pkg/lib/template/template_loader_test.go +++ b/pkg/lib/template/template_loader_test.go @@ -82,7 +82,7 @@ func TestBigIntRender(t *testing.T) { inputNumber := "132132132182323" - resp, _ := api.NewResponse(200, nil, nil, nil, strings.NewReader(fmt.Sprintf(`{"bigINT":%s}`, inputNumber)), nil, api.ResponseFormat{}) + resp, _ := api.NewResponse(200, nil, nil, nil, strings.NewReader(fmt.Sprintf(`{"bigINT":%s}`, inputNumber)), api.ResponseFormat{}) respJson, _ := resp.ServerResponseToJsonString(false) store.SetWithQjson(respJson, map[string]string{"testINT": "body.bigINT"}) @@ -415,7 +415,6 @@ func Test_DataStore_QJson(t *testing.T) { "flob" ] }`), - nil, api.ResponseFormat{}, ) store := datastore.NewStore(false) diff --git a/test/control/header/manifest.json b/test/control/header/manifest.json index 644b1ef7..a612b377 100644 --- a/test/control/header/manifest.json +++ b/test/control/header/manifest.json @@ -43,21 +43,6 @@ } } }, - { - "name": "check HTTP header using control, use reverse_test_result", - "request": { - "server_url": "http://localhost:9999", - "endpoint": "bounce-json", - "method": "POST" - }, - "response": { - // check number of HTTP headers, should always be > 0 - "header:control": { - "element_count": 0 - } - }, - "reverse_test_result": true - }, { "name": "check value in HTTP header using control, use reverse_test_result", "request": { diff --git a/test/datastore/check.json b/test/datastore/check.json index 40de596c..4cca4797 100644 --- a/test/datastore/check.json +++ b/test/datastore/check.json @@ -18,11 +18,7 @@ {"some": "data"}, {"some": ["more", "data"]}, {"some": "data"} - ], - "body:control": { - "order_matters": true, - "no_extra": true - } + ] } } } diff --git a/test/response/header/manifest.json b/test/response/header/manifest.json new file mode 100644 index 00000000..4771746f --- /dev/null +++ b/test/response/header/manifest.json @@ -0,0 +1,84 @@ +{ + "http_server": { + "addr": ":9999", + "dir": "../_res", + "testmode": false + }, + "name": "response header: format header_flat", + "tests": [ + { + "name": "test response header_flat", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header": { + "Content-Type": [ + "text/plain; charset=utf-8" + ] + }, + "header_flat": { + "Content-Type": "text/plain; charset=utf-8" + } + } + }, + { + "name": "test response header_flat (check for a failing test with a reverse result)", + "reverse_test_result": true, + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header_flat": { + "Content-Length": "foo", + "Content-Type": "bar" + } + } + }, + { + "name": "test response header_flat: control for header values", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header_flat": { + "Content-Length:control": { + "match": "^\\d+$" + }, + "Content-Type:control": { + "match": "^text/plain;.*" + } + } + } + }, + { + "name": "test response header_flat: control for header values (check for a failing test with a reverse result)", + "reverse_test_result": true, + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header_flat": { + "Content-Length:control": { + "match": "foo" + }, + "Content-Type:control": { + "match": "bar" + } + } + } + } + ] +} \ No newline at end of file From 536312f908048169d4f88f40c51d56ed1b811db6 Mon Sep 17 00:00:00 2001 From: Philipp Hempel Date: Mon, 28 Oct 2024 12:27:16 +0100 Subject: [PATCH 2/6] replaced usage of obsolete ioutil functions --- api_testsuite.go | 4 ++-- http_server.go | 3 +-- internal/httpproxy/store.go | 4 ++-- pkg/lib/api/request.go | 3 +-- pkg/lib/api/response.go | 3 +-- pkg/lib/report/report.go | 4 ++-- pkg/lib/template/template_loader.go | 2 +- 7 files changed, 10 insertions(+), 13 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index fc4b5f3c..b05bfbe9 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -3,7 +3,7 @@ package main import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "os" @@ -442,7 +442,7 @@ func (ats *Suite) loadManifest() ([]byte, error) { } defer manifestFile.Close() - manifestTmpl, err := ioutil.ReadAll(manifestFile) + manifestTmpl, err := io.ReadAll(manifestFile) if err != nil { return res, fmt.Errorf("error loading manifest (%s): %s", ats.manifestPath, err) } diff --git a/http_server.go b/http_server.go index 02250147..f246a3ee 100644 --- a/http_server.go +++ b/http_server.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "io" - "io/ioutil" "net/http" "net/url" "path/filepath" @@ -155,7 +154,7 @@ func bounceJSON(w http.ResponseWriter, r *http.Request) { bodyJSON, errorBody any ) - bodyBytes, err = ioutil.ReadAll(r.Body) + bodyBytes, err = io.ReadAll(r.Body) if utf8.Valid(bodyBytes) { if len(bodyBytes) > 0 { diff --git a/internal/httpproxy/store.go b/internal/httpproxy/store.go index 99ead104..af50c676 100644 --- a/internal/httpproxy/store.go +++ b/internal/httpproxy/store.go @@ -3,7 +3,7 @@ package httpproxy import ( "encoding/json" "fmt" - "io/ioutil" + "io" "net/http" "net/url" "strconv" @@ -63,7 +63,7 @@ func (st *store) write(w http.ResponseWriter, r *http.Request) { offset := len(st.Data) if r.Body != nil { - reqData.Body, err = ioutil.ReadAll(r.Body) + reqData.Body, err = io.ReadAll(r.Body) if err != nil { handlerutil.RespondWithErr(w, http.StatusInternalServerError, errors.Errorf("Could not read request body: %s", err)) return diff --git a/pkg/lib/api/request.go b/pkg/lib/api/request.go index a3a4619c..2e1bd56c 100755 --- a/pkg/lib/api/request.go +++ b/pkg/lib/api/request.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "net/http/httputil" "net/url" @@ -293,7 +292,7 @@ func (request Request) ToString(curl bool) (res string) { // return r.Replace(curl.String()) } - _, _ = io.Copy(ioutil.Discard, httpRequest.Body) + _, _ = io.Copy(io.Discard, httpRequest.Body) _ = httpRequest.Body.Close() curl, _ := http2curl.GetCurlCommand(httpRequest) diff --git a/pkg/lib/api/response.go b/pkg/lib/api/response.go index 1d095c26..92f75504 100755 --- a/pkg/lib/api/response.go +++ b/pkg/lib/api/response.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "strings" "time" @@ -102,7 +101,7 @@ func NewResponse(statusCode int, } if body != nil { start := time.Now() - res.Body, err = ioutil.ReadAll(body) + res.Body, err = io.ReadAll(body) if err != nil { return res, err } diff --git a/pkg/lib/report/report.go b/pkg/lib/report/report.go index a8f072a9..c9ee5af9 100755 --- a/pkg/lib/report/report.go +++ b/pkg/lib/report/report.go @@ -2,7 +2,7 @@ package report import ( "fmt" - "io/ioutil" + "os" "sync" "time" @@ -196,7 +196,7 @@ func (r *Report) WriteToFile(reportFile, reportFormat string) error { parsingFunction = ParseJSONResult } - err := ioutil.WriteFile(reportFile, r.GetTestResult(parsingFunction), 0644) + err := os.WriteFile(reportFile, r.GetTestResult(parsingFunction), 0644) if err != nil { logrus.Errorf("Could not save report into file: %s", err) return err diff --git a/pkg/lib/template/template_loader.go b/pkg/lib/template/template_loader.go index 4074e35f..160a07a6 100644 --- a/pkg/lib/template/template_loader.go +++ b/pkg/lib/template/template_loader.go @@ -128,7 +128,7 @@ func (loader *Loader) Render( // if err != nil { // return nil, err // } - // fileBytes, err := ioutil.ReadAll(file) + // fileBytes, err := io.ReadAll(file) // if err != nil { // return nil, err // } From 27423dc3d5fa37fc3712121f16d27ccbc71c65c2 Mon Sep 17 00:00:00 2001 From: Philipp Hempel Date: Mon, 28 Oct 2024 12:40:39 +0100 Subject: [PATCH 3/6] replaced errors.Wrap and other obsolete functions --- api_testsuite.go | 5 ++--- internal/httpproxy/store.go | 12 +++++------- pkg/lib/api/request.go | 4 +--- pkg/lib/api/response.go | 16 +++++++--------- pkg/lib/compare/comparison_functions.go | 5 ++--- pkg/lib/csv/csv.go | 4 +--- pkg/lib/template/template_funcs.go | 10 +++++----- pkg/lib/template/template_loader.go | 23 +++++++++++------------ pkg/lib/util/oauth.go | 4 ++-- pkg/lib/util/util.go | 15 +++++++-------- 10 files changed, 43 insertions(+), 55 deletions(-) diff --git a/api_testsuite.go b/api_testsuite.go index b05bfbe9..8c6ebd3c 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -14,7 +14,6 @@ import ( "sync/atomic" "time" - "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/programmfabrik/apitest/internal/httpproxy" @@ -105,7 +104,7 @@ func NewTestSuite(config TestToolConfig, manifestPath string, manifestDir string if httpServerReplaceHost != "" { _, err = url.Parse("//" + httpServerReplaceHost) if err != nil { - return nil, errors.Wrap(err, "set http_server_host failed (command argument)") + return nil, fmt.Errorf("set http_server_host failed (command argument): %w", err) } } if suitePreload.HttpServer != nil { @@ -116,7 +115,7 @@ func NewTestSuite(config TestToolConfig, manifestPath string, manifestDir string // We need to append it as the golang URL parser is not smart enough to differenciate between hostname and protocol _, err = url.Parse("//" + preloadHTTPAddrStr) if err != nil { - return nil, errors.Wrap(err, "set http_server_host failed (manifesr addr)") + return nil, fmt.Errorf("set http_server_host failed (manifesr addr): %w", err) } } suitePreload.HTTPServerHost = httpServerReplaceHost diff --git a/internal/httpproxy/store.go b/internal/httpproxy/store.go index af50c676..828cc061 100644 --- a/internal/httpproxy/store.go +++ b/internal/httpproxy/store.go @@ -8,8 +8,6 @@ import ( "net/url" "strconv" - "github.com/pkg/errors" - "github.com/programmfabrik/apitest/internal/handlerutil" ) @@ -65,7 +63,7 @@ func (st *store) write(w http.ResponseWriter, r *http.Request) { if r.Body != nil { reqData.Body, err = io.ReadAll(r.Body) if err != nil { - handlerutil.RespondWithErr(w, http.StatusInternalServerError, errors.Errorf("Could not read request body: %s", err)) + handlerutil.RespondWithErr(w, http.StatusInternalServerError, fmt.Errorf("Could not read request body: %w", err)) return } } @@ -76,7 +74,7 @@ func (st *store) write(w http.ResponseWriter, r *http.Request) { Offset int `json:"offset"` }{offset}) if err != nil { - handlerutil.RespondWithErr(w, http.StatusInternalServerError, errors.Errorf("Could not encode response: %s", err)) + handlerutil.RespondWithErr(w, http.StatusInternalServerError, fmt.Errorf("Could not encode response: %w", err)) } } @@ -94,14 +92,14 @@ func (st *store) read(w http.ResponseWriter, r *http.Request) { if offsetStr != "" { offset, err = strconv.Atoi(offsetStr) if err != nil { - handlerutil.RespondWithErr(w, http.StatusBadRequest, errors.Errorf("Invalid offset %s", offsetStr)) + handlerutil.RespondWithErr(w, http.StatusBadRequest, fmt.Errorf("Invalid offset %s", offsetStr)) return } } count := len(st.Data) if offset >= count { - handlerutil.RespondWithErr(w, http.StatusBadRequest, errors.Errorf("Offset (%d) is higher than count (%d)", offset, count)) + handlerutil.RespondWithErr(w, http.StatusBadRequest, fmt.Errorf("Offset (%d) is higher than count (%d)", offset, count)) return } @@ -126,6 +124,6 @@ func (st *store) read(w http.ResponseWriter, r *http.Request) { _, err = w.Write(req.Body) if err != nil { - handlerutil.RespondWithErr(w, http.StatusInternalServerError, errors.Errorf("Could not encode response: %s", err)) + handlerutil.RespondWithErr(w, http.StatusInternalServerError, fmt.Errorf("Could not encode response: %w", err)) } } diff --git a/pkg/lib/api/request.go b/pkg/lib/api/request.go index 2e1bd56c..0ef79383 100755 --- a/pkg/lib/api/request.go +++ b/pkg/lib/api/request.go @@ -12,8 +12,6 @@ import ( "strings" "time" - "github.com/pkg/errors" - "github.com/moul/http2curl" "github.com/programmfabrik/apitest/pkg/lib/datastore" "github.com/programmfabrik/apitest/pkg/lib/util" @@ -84,7 +82,7 @@ func (request Request) buildHttpRequest() (req *http.Request, err error) { reqUrl, err := url.Parse(requestUrl) if err != nil { - return nil, errors.Wrapf(err, "Unable to buildHttpRequest with URL %q", requestUrl) + return nil, fmt.Errorf("Unable to buildHttpRequest with URL %q: %w", requestUrl, err) } // Note that buildPolicy may return a file handle that needs to be diff --git a/pkg/lib/api/response.go b/pkg/lib/api/response.go index 92f75504..1aa79f12 100755 --- a/pkg/lib/api/response.go +++ b/pkg/lib/api/response.go @@ -12,8 +12,6 @@ import ( "time" "unicode/utf8" - "github.com/pkg/errors" - "github.com/programmfabrik/apitest/pkg/lib/csv" "github.com/programmfabrik/apitest/pkg/lib/util" "github.com/programmfabrik/golib" @@ -158,7 +156,7 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm if responseFormat.PreProcess != nil { resp, err = responseFormat.PreProcess.RunPreProcess(response) if err != nil { - return res, errors.Wrap(err, "Could not pre process response") + return res, fmt.Errorf("Could not pre process response: %w", err) } } else { resp = response @@ -168,17 +166,17 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm case "xml", "xml2": bodyData, err = util.Xml2Json(resp.Body, responseFormat.Type) if err != nil { - return res, errors.Wrap(err, "Could not marshal xml to json") + return res, fmt.Errorf("Could not marshal xml to json: %w", err) } case "html": bodyData, err = util.Html2Json(resp.Body) if err != nil { - return res, errors.Wrap(err, "Could not marshal html to json") + return res, fmt.Errorf("Could not marshal html to json: %w", err) } case "xhtml": bodyData, err = util.Xhtml2Json(resp.Body) if err != nil { - return res, errors.Wrap(err, "Could not marshal xhtml to json") + return res, fmt.Errorf("Could not marshal xhtml to json: %w", err) } case "csv": runeComma := ',' @@ -188,12 +186,12 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm csvData, err := csv.GenericCSVToMap(resp.Body, runeComma) if err != nil { - return res, errors.Wrap(err, "Could not parse csv") + return res, fmt.Errorf("Could not parse csv: %w", err) } bodyData, err = json.Marshal(csvData) if err != nil { - return res, errors.Wrap(err, "Could not marshal csv to json") + return res, fmt.Errorf("Could not marshal csv to json: %w", err) } case "binary": // We have another file format (binary). We thereby take the md5 Hash of the body and compare that one @@ -204,7 +202,7 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm } bodyData, err = json.Marshal(JsonObject) if err != nil { - return res, errors.Wrap(err, "Could not marshal body with md5sum to json") + return res, fmt.Errorf("Could not marshal body with md5sum to json: %w", err) } case "": // no specific format, we assume a json, and thereby try to unmarshal it into our body diff --git a/pkg/lib/compare/comparison_functions.go b/pkg/lib/compare/comparison_functions.go index 45c5a215..509a3a5a 100755 --- a/pkg/lib/compare/comparison_functions.go +++ b/pkg/lib/compare/comparison_functions.go @@ -8,7 +8,6 @@ import ( "strconv" "strings" - "github.com/pkg/errors" "github.com/programmfabrik/apitest/pkg/lib/util" "github.com/programmfabrik/golib" ) @@ -361,11 +360,11 @@ func arrayComparison(left, right util.JsonArray, currControl ComparisonContext, leftJson, err := golib.JsonBytesIndent(left, "", " ") if err != nil { - return CompareResult{}, errors.Wrap(err, "Could not marshal expected array") + return CompareResult{}, fmt.Errorf("Could not marshal expected array: %w", err) } rightJson, err := golib.JsonBytesIndent(right, "", " ") if err != nil { - return CompareResult{}, errors.Wrap(err, "Could not marshal actual array") + return CompareResult{}, fmt.Errorf("Could not marshal actual array: %w", err) } res.Failures = append(res.Failures, CompareFailure{"", fmt.Sprintf("[arrayComparison] length of expected response (%d) > length of actual response (%d)\nExpected response:\n%s\nActual response:\n%s\n", len(left), len(right), string(leftJson), string(rightJson))}) diff --git a/pkg/lib/csv/csv.go b/pkg/lib/csv/csv.go index a3a723a8..663b2ba4 100644 --- a/pkg/lib/csv/csv.go +++ b/pkg/lib/csv/csv.go @@ -8,8 +8,6 @@ import ( "io" "strconv" "strings" - - "github.com/pkg/errors" ) // Get information @@ -25,7 +23,7 @@ func CSVToMap(inputCSV []byte, comma rune) ([]map[string]any, error) { records, err := renderCSV(bytes.NewReader(inputCSV), comma) if err != nil { - return nil, errors.Wrap(err, "CSVToMap.renderCSV") + return nil, fmt.Errorf("CSVToMap.renderCSV: %w", err) } records = removeEmptyRowsAndComments(records) diff --git a/pkg/lib/template/template_funcs.go b/pkg/lib/template/template_funcs.go index 80b676f7..62f4f4b6 100644 --- a/pkg/lib/template/template_funcs.go +++ b/pkg/lib/template/template_funcs.go @@ -86,7 +86,7 @@ func pivotRows(key, typ string, rows []map[string]any) (sheet []map[string]any, case "string", "int64", "float64", "number", "json": // supported default: - return nil, errors.Errorf("type %q not supported", sheetType) + return nil, fmt.Errorf("type %q not supported", sheetType) } for kI, vI := range row { @@ -325,13 +325,13 @@ func divide(b, a any) (any, error) { func fileReadInternal(pathOrURL, rootDir string) ([]byte, error) { file, err := util.OpenFileOrUrl(pathOrURL, rootDir) if err != nil { - return nil, errors.Wrapf(err, "fileReadInternal: %q", pathOrURL) + return nil, fmt.Errorf("fileReadInternal: %q: %w", pathOrURL, err) } defer file.Close() data, err := io.ReadAll(file) if err != nil { - return nil, errors.Wrapf(err, "fileReadInternal: %q", pathOrURL) + return nil, fmt.Errorf("fileReadInternal: %q: %w", pathOrURL, err) } return data, nil } @@ -350,7 +350,7 @@ func loadFileAndRender(rootDir string, loader *Loader) any { } data, err = loader.Render(data, filepath.Dir(filepath.Join(rootDir, path)), tmplParams) if err != nil { - return "", errors.Wrapf(err, "Render error in file %q", path) + return "", fmt.Errorf("Render error in file %q: %w", path, err) } return string(data), nil } @@ -387,7 +387,7 @@ func loadFileCSV(rootDir string) any { } data, err := csv.CSVToMap(fileBytes, delimiter) if err != nil { - return data, errors.Wrapf(err, "CSV map error in file %q", path) + return data, fmt.Errorf("CSV map error in file %q: %w", path, err) } return data, err } diff --git a/pkg/lib/template/template_loader.go b/pkg/lib/template/template_loader.go index 160a07a6..4075314d 100644 --- a/pkg/lib/template/template_loader.go +++ b/pkg/lib/template/template_loader.go @@ -15,7 +15,6 @@ import ( "text/template" "github.com/Masterminds/sprig/v3" - "github.com/pkg/errors" "github.com/programmfabrik/apitest/pkg/lib/datastore" "github.com/programmfabrik/golib" "github.com/sirupsen/logrus" @@ -192,7 +191,7 @@ func (loader *Loader) Render( bytes, err := util.Xml2Json(fileBytes, "xml2") if err != nil { - return "", errors.Wrap(err, "Could not marshal xml to json") + return "", fmt.Errorf("Could not marshal xml to json: %w", err) } return string(bytes), nil @@ -205,7 +204,7 @@ func (loader *Loader) Render( bytes, err := util.Xhtml2Json(fileBytes) if err != nil { - return "", errors.Wrap(err, "Could not marshal xhtml to json") + return "", fmt.Errorf("Could not marshal xhtml to json: %w", err) } return string(bytes), nil @@ -218,7 +217,7 @@ func (loader *Loader) Render( bytes, err := util.Html2Json(fileBytes) if err != nil { - return "", errors.Wrap(err, "Could not marshal html to json") + return "", fmt.Errorf("Could not marshal html to json: %w", err) } return string(bytes), nil @@ -373,7 +372,7 @@ func (loader *Loader) Render( // println("client", client, login, password) oAuthClient, ok := loader.OAuthClient[client] if !ok { - return nil, errors.Errorf("OAuth client %q not configured", client) + return nil, fmt.Errorf("OAuth client %q not configured", client) } return oAuthClient.GetPasswordCredentialsAuthToken(login, password) @@ -382,7 +381,7 @@ func (loader *Loader) Render( "oauth2_client_token": func(client string) (tok *oauth2.Token, err error) { oAuthClient, ok := loader.OAuthClient[client] if !ok { - return nil, errors.Errorf("OAuth client %q not configured", client) + return nil, fmt.Errorf("OAuth client %q not configured", client) } return oAuthClient.GetClientCredentialsAuthToken() @@ -390,7 +389,7 @@ func (loader *Loader) Render( "oauth2_code_token": func(client string, params ...string) (tok *oauth2.Token, err error) { oAuthClient, ok := loader.OAuthClient[client] if !ok { - return nil, errors.Errorf("OAuth client %q not configured", client) + return nil, fmt.Errorf("OAuth client %q not configured", client) } return oAuthClient.GetCodeAuthToken(params...) @@ -398,7 +397,7 @@ func (loader *Loader) Render( "oauth2_implicit_token": func(client string, params ...string) (tok *oauth2.Token, err error) { oAuthClient, ok := loader.OAuthClient[client] if !ok { - return nil, errors.Errorf("OAuth client %q not configured", client) + return nil, fmt.Errorf("OAuth client %q not configured", client) } return oAuthClient.GetAuthToken(params...) @@ -406,7 +405,7 @@ func (loader *Loader) Render( "oauth2_client": func(client string) (c *util.OAuthClientConfig, err error) { oAuthClient, ok := loader.OAuthClient[client] if !ok { - return nil, errors.Errorf("OAuth client %s not configured", client) + return nil, fmt.Errorf("OAuth client %s not configured", client) } return &oAuthClient, nil @@ -414,7 +413,7 @@ func (loader *Loader) Render( "oauth2_basic_auth": func(client string) (string, error) { oAuthClient, ok := loader.OAuthClient[client] if !ok { - return "", errors.Errorf("OAuth client %s not configured", client) + return "", fmt.Errorf("OAuth client %s not configured", client) } return "Basic " + base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", oAuthClient.Client, oAuthClient.Secret))), nil @@ -449,10 +448,10 @@ func (loader *Loader) Render( w = "v0.0.0" } if !semver.IsValid(v) { - return 0, errors.Errorf("version string %s is invalid", v) + return 0, fmt.Errorf("version string %s is invalid", v) } if !semver.IsValid(w) { - return 0, errors.Errorf("version string %s is invalid", w) + return 0, fmt.Errorf("version string %s is invalid", w) } return semver.Compare(v, w), nil }, diff --git a/pkg/lib/util/oauth.go b/pkg/lib/util/oauth.go index b38896d4..e3c3aec0 100644 --- a/pkg/lib/util/oauth.go +++ b/pkg/lib/util/oauth.go @@ -2,13 +2,13 @@ package util import ( "context" + "fmt" "net/http" "net/url" "time" "log" - "github.com/pkg/errors" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" ) @@ -122,7 +122,7 @@ func (c OAuthClientConfig) getRedirectURL(params ...string) (*url.URL, error) { return nil, err } if res.StatusCode != http.StatusOK { - return nil, errors.Errorf("No proper status after redirect returned: %s (%d)", res.Status, res.StatusCode) + return nil, fmt.Errorf("No proper status after redirect returned: %s (%d)", res.Status, res.StatusCode) } return res.Request.URL, nil } diff --git a/pkg/lib/util/util.go b/pkg/lib/util/util.go index df0fc4f3..d5c87f69 100644 --- a/pkg/lib/util/util.go +++ b/pkg/lib/util/util.go @@ -10,7 +10,6 @@ import ( "github.com/PuerkitoBio/goquery" "github.com/clbanning/mxj" - "github.com/pkg/errors" "github.com/programmfabrik/golib" "golang.org/x/net/html" ) @@ -67,16 +66,16 @@ func Xml2Json(rawXml []byte, format string) ([]byte, error) { case "xml2": mv, err = mxj.NewMapXml(replacedXML) default: - return []byte{}, errors.Errorf("Unknown format %s", format) + return []byte{}, fmt.Errorf("Unknown format %s", format) } if err != nil { - return []byte{}, errors.Wrap(err, "Could not parse xml") + return []byte{}, fmt.Errorf("Could not parse xml: %w", err) } jsonStr, err := mv.JsonIndent("", " ") if err != nil { - return []byte{}, errors.Wrap(err, "Could not convert to json") + return []byte{}, fmt.Errorf("Could not convert to json: %w", err) } return jsonStr, nil } @@ -90,12 +89,12 @@ func Xhtml2Json(rawXhtml []byte) ([]byte, error) { mv, err = mxj.NewMapXml(rawXhtml) if err != nil { - return []byte{}, errors.Wrap(err, "Could not parse xhtml") + return []byte{}, fmt.Errorf("Could not parse xhtml: %w", err) } jsonStr, err := mv.JsonIndent("", " ") if err != nil { - return []byte{}, errors.Wrap(err, "Could not convert to json") + return []byte{}, fmt.Errorf("Could not convert to json: %w", err) } return jsonStr, nil } @@ -109,7 +108,7 @@ func Html2Json(rawHtml []byte) ([]byte, error) { htmlDoc, err = goquery.NewDocumentFromReader(bytes.NewReader(rawHtml)) if err != nil { - return []byte{}, errors.Wrap(err, "Could not parse html") + return []byte{}, fmt.Errorf("Could not parse html: %w", err) } htmlData := map[string]any{} @@ -125,7 +124,7 @@ func Html2Json(rawHtml []byte) ([]byte, error) { jsonStr, err := golib.JsonBytesIndent(htmlData, "", " ") if err != nil { - return []byte{}, errors.Wrap(err, "Could not convert html to json") + return []byte{}, fmt.Errorf("Could not convert html to json: %w", err) } return jsonStr, nil From 66c865bb5afa190bc7786792fd54d49ffc6634a1 Mon Sep 17 00:00:00 2001 From: Martin Rode Date: Thu, 31 Oct 2024 11:10:09 +0100 Subject: [PATCH 4/6] support header string + array for request & response Also, support ":control" in header for the flattened version. see #73980 --- README.md | 24 +++-- api_testcase.go | 11 +- api_testsuite.go | 6 +- pkg/lib/api/request.go | 31 ++++-- pkg/lib/api/response.go | 129 ++++++++++++++--------- pkg/lib/api/response_test.go | 6 +- pkg/lib/template/template_loader_test.go | 4 +- test/datastore/check.json | 12 ++- test/response/header/header1.json | 54 ++++++++++ test/response/header/header2.json | 16 +++ test/response/header/header3.json | 19 ++++ test/response/header/header4.json | 20 ++++ test/response/header/header5.json | 16 +++ test/response/header/header6.json | 43 ++++++++ test/response/header/manifest.json | 82 ++------------ 15 files changed, 316 insertions(+), 157 deletions(-) create mode 100644 test/response/header/header1.json create mode 100644 test/response/header/header2.json create mode 100644 test/response/header/header3.json create mode 100644 test/response/header/header4.json create mode 100644 test/response/header/header5.json create mode 100644 test/response/header/header6.json diff --git a/README.md b/README.md index 0593a88b..10fe8e12 100644 --- a/README.md +++ b/README.md @@ -224,14 +224,14 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * "query_params_from_store": { "format": "formatFromDatastore", // If the datastore key starts with an ?, wo do not throw an error if the key could not be found, but just - // do not set the query param. If the key "a" is not found it datastore, the queryparameter test will not be set + // do not set the query param. If the key "a" is not found it datastore, the query parameter test will not be set "test": "?a" }, // Additional headers that should be added to the request "header": { "header1": "value", - "header2": "value" + "header2": ["value1", "value2"] }, // Cookies can be added to the request @@ -270,7 +270,7 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * } ], - // With header_from_you set a header to the value of the dat astore field + // With header_from_store you set a header to the value of the datastore field // In this example we set the "Content-Type" header to the value "application/json" // As "application/json" is stored as string in the datastore on index "contentType" "header_from_store": { @@ -298,13 +298,22 @@ Manifest is loaded as **template**, so you can use variables, Go **range** and * // Expected http status code. See api documentation vor the right ones "statuscode": 200, - // If you expect certain response headers, you can define them here. A single key can have mulitble headers (as defined in rfc2616) + // If you expect certain response headers, you can define them here. A single key can have multiple headers (as defined in rfc2616) "header": { "key1": [ "val1", "val2", "val3" ], + + // Headers sharing the same key are concatenated using ";", if the comparison value is a simple string, + // thus "key1" can also be checked like this: + "key1": "val1;val2;val3" + + // :control in header is always applied to the flat format + "key1:control": { + // see below, this is not applied against the array + }, "x-easydb-token": [ "csdklmwerf8ßwji02kopwfjko2" ] @@ -1931,10 +1940,7 @@ The datastore stores all responses in a list. We can retrieve the response (as a { "statuscode": 200, "header": { - "foo": [ - "bar", - "baz" - ] + "foo": "bar;baz" }, "body": "..." } @@ -2658,7 +2664,7 @@ The endpoint `bounce` returns the binary of the request body, as well as the req "param1": "abc" }, "header": { - "header1": 123 + "header1": "123" }, "body": { "file": "@path/to/file.jpg" diff --git a/api_testcase.go b/api_testcase.go index 42bdd6ce..ea032a8e 100644 --- a/api_testcase.go +++ b/api_testcase.go @@ -49,8 +49,7 @@ type Case struct { index int dataStore *datastore.Datastore - standardHeader map[string]*string - headerFlat map[string]*string + standardHeader map[string]any // can be string or []string standardHeaderFromStore map[string]string ServerURL string `json:"server_url"` @@ -490,7 +489,7 @@ func (testCase Case) responsesEqual(expected, got api.Response) (compare.Compare if err != nil { return compare.CompareResult{}, fmt.Errorf("error loading expected generic json: %s", err) } - if len(expected.Body) == 0 { + if len(expected.Body) == 0 && len(expected.BodyControl) == 0 { expected.Format.IgnoreBody = true } else { expected.Format.IgnoreBody = false @@ -524,10 +523,10 @@ func (testCase Case) loadRequestSerialization() (api.Request, error) { spec.ServerURL = testCase.ServerURL } if len(spec.Headers) == 0 { - spec.Headers = make(map[string]*string) + spec.Headers = make(map[string]any) } for k, v := range testCase.standardHeader { - if spec.Headers[k] == nil { + if _, exist := spec.Headers[k]; !exist { spec.Headers[k] = v } } @@ -536,7 +535,7 @@ func (testCase Case) loadRequestSerialization() (api.Request, error) { spec.HeaderFromStore = make(map[string]string) } for k, v := range testCase.standardHeaderFromStore { - if spec.HeaderFromStore[k] == "" { + if _, exist := spec.HeaderFromStore[k]; !exist { spec.HeaderFromStore[k] = v } } diff --git a/api_testsuite.go b/api_testsuite.go index 8c6ebd3c..bd3ad3a0 100644 --- a/api_testsuite.go +++ b/api_testsuite.go @@ -42,9 +42,8 @@ type Suite struct { Tests []any `json:"tests"` Store map[string]any `json:"store"` - StandardHeader map[string]*string `yaml:"header" json:"header"` - HeaderFlat map[string]*string `yaml:"header_flat" json:"header_flat"` - StandardHeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` + StandardHeader map[string]any `yaml:"header" json:"header"` + StandardHeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` Config TestToolConfig datastore *datastore.Datastore @@ -399,7 +398,6 @@ func (ats *Suite) runLiteralTest( test.index = index test.dataStore = ats.datastore test.standardHeader = ats.StandardHeader - test.headerFlat = ats.HeaderFlat test.standardHeaderFromStore = ats.StandardHeaderFromStore if test.LogNetwork == nil { test.LogNetwork = &ats.Config.LogNetwork diff --git a/pkg/lib/api/request.go b/pkg/lib/api/request.go index 0ef79383..5d484897 100755 --- a/pkg/lib/api/request.go +++ b/pkg/lib/api/request.go @@ -46,7 +46,7 @@ type Request struct { NoRedirect bool `yaml:"no_redirect" json:"no_redirect"` QueryParams map[string]any `yaml:"query_params" json:"query_params"` QueryParamsFromStore map[string]string `yaml:"query_params_from_store" json:"query_params_from_store"` - Headers map[string]*string `yaml:"header" json:"header"` + Headers map[string]any `yaml:"header" json:"header"` HeaderFromStore map[string]string `yaml:"header_from_store" json:"header_from_store"` Cookies map[string]*RequestCookie `yaml:"cookies" json:"cookies"` SetCookies []*Cookie `yaml:"header-x-test-set-cookie" json:"header-x-test-set-cookie"` @@ -202,12 +202,25 @@ func (request Request) buildHttpRequest() (req *http.Request, err error) { } for key, val := range request.Headers { - if *val == "" { - //Unset header explicit - req.Header.Del(key) - } else { - //ADD header - req.Header.Set(key, *val) + switch v := val.(type) { + case string: + if v == "" { + req.Header.Del(key) + } else { + req.Header.Set(key, v) + } + case []any: + vArr := []string{} + for _, vOne := range v { + vOneS, isString := vOne.(string) + if !isString { + return nil, fmt.Errorf("unsupported header %q value %T: %v", key, vOne, vOne) + } + vArr = append(vArr, vOneS) + } + req.Header[key] = vArr + default: + return nil, fmt.Errorf("unsupported header value %T: %v", val, val) } } @@ -344,9 +357,9 @@ func (request Request) Send() (response Response, err error) { if err != nil { return response, err } - response, err = NewResponse(httpResponse.StatusCode, header, nil, httpResponse.Cookies(), httpResponse.Body, ResponseFormat{}) + response, err = NewResponse(httpResponse.StatusCode, header, httpResponse.Cookies(), httpResponse.Body, nil, ResponseFormat{}) if err != nil { - return response, fmt.Errorf("error constructing response from http response") + return response, fmt.Errorf("error constructing response from http response: %w", err) } response.ReqDur = elapsedTime return response, err diff --git a/pkg/lib/api/response.go b/pkg/lib/api/response.go index 1aa79f12..5ea39663 100755 --- a/pkg/lib/api/response.go +++ b/pkg/lib/api/response.go @@ -18,32 +18,18 @@ import ( ) type Response struct { - StatusCode int - Headers map[string]any - HeaderFlat map[string]any - Cookies []*http.Cookie - Body []byte - Format ResponseFormat + StatusCode int + Headers map[string]any + HeaderFlat map[string]any // ":control" is an object, so we must use "any" here + Cookies []*http.Cookie + Body []byte + BodyControl util.JsonObject + Format ResponseFormat ReqDur time.Duration BodyLoadDur time.Duration } -func (res Response) SerializeHeaderFlat() (headers map[string]any, err error) { - headers = map[string]any{} - for k, h := range res.Headers { - switch v := h.(type) { - case string: - headers[k] = h - case nil: - headers[k] = "" - case []string: - headers[k] = strings.Join(v, "; ") - } - } - return headers, nil -} - func HttpHeaderToMap(header http.Header) (headers map[string]any, err error) { headers = map[string]any{} for k, h := range header { @@ -66,12 +52,17 @@ type Cookie struct { } type ResponseSerialization struct { - StatusCode int `yaml:"statuscode" json:"statuscode"` - Headers map[string]any `yaml:"header" json:"header,omitempty"` - HeaderFlat map[string]any `yaml:"header_flat" json:"header_flat,omitempty"` - Cookies map[string]Cookie `yaml:"cookie" json:"cookie,omitempty"` - Body any `yaml:"body" json:"body,omitempty"` - Format ResponseFormat `yaml:"format" json:"format,omitempty"` + StatusCode int `yaml:"statuscode" json:"statuscode"` + Headers map[string]any `yaml:"header" json:"header,omitempty"` + Cookies map[string]Cookie `yaml:"cookie" json:"cookie,omitempty"` + Body any `yaml:"body" json:"body,omitempty"` + BodyControl util.JsonObject `yaml:"body:control" json:"body:control,omitempty"` + Format ResponseFormat `yaml:"format" json:"format,omitempty"` +} + +type responseSerializationInternal struct { + ResponseSerialization + HeaderFlat map[string]any `json:"header_flat,omitemty"` } type ResponseFormat struct { @@ -84,18 +75,55 @@ type ResponseFormat struct { } func NewResponse(statusCode int, - headers map[string]any, - headerFlat map[string]any, + headersAny map[string]any, cookies []*http.Cookie, body io.Reader, + bodyControl util.JsonObject, bodyFormat ResponseFormat, ) (res Response, err error) { + + headerFlat := map[string]any{} + headers := map[string]any{} + + // parse headers and set HeaderFlat if the values are string + for key, value := range headersAny { + switch v := value.(type) { + case string: + headerFlat[key] = v + continue + case []any: + headerS := []string{} + for _, item := range v { + switch v2 := item.(type) { + case string: + headerS = append(headerS, v2) + continue + default: + return res, fmt.Errorf("unknown type %T in header %q", v2, key) + } + } + headers[key] = v + continue + case []string: + headers[key] = v + continue + case map[string]any: // check if that is a control + if strings.HasSuffix(key, ":control") { + headerFlat[key] = v + continue + } + } + // all valid cases continue above + return res, fmt.Errorf("unknown type %T in header %q", value, key) + } + res = Response{ - StatusCode: statusCode, - Headers: headers, - HeaderFlat: headerFlat, - Cookies: cookies, - Format: bodyFormat, + StatusCode: statusCode, + Headers: headers, + BodyControl: bodyControl, + HeaderFlat: headerFlat, + Cookies: cookies, + Format: bodyFormat, } if body != nil { start := time.Now() @@ -141,7 +169,7 @@ func NewResponseFromSpec(spec ResponseSerialization) (res Response, err error) { } } - return NewResponse(spec.StatusCode, spec.Headers, spec.HeaderFlat, cookies, body, spec.Format) + return NewResponse(spec.StatusCode, spec.Headers, cookies, body, spec.BodyControl, spec.Format) } // ServerResponseToGenericJSON parse response from server. convert xml, csv, binary to json if necessary @@ -211,15 +239,19 @@ func (response Response) ServerResponseToGenericJSON(responseFormat ResponseForm return res, fmt.Errorf("Invalid response format '%s'", responseFormat.Type) } - headerFlat, err := resp.SerializeHeaderFlat() - if err != nil { - return res, err - } - responseJSON := ResponseSerialization{ - StatusCode: resp.StatusCode, - Headers: resp.Headers, - HeaderFlat: headerFlat, + headerFlat := map[string]any{} + headersAny := map[string]any{} + for key, value := range resp.Headers { + headersAny[key] = value + values := value.([]string) // this must be []string, if not this panics + headerFlat[key] = strings.Join(values, ";") } + + responseJSON := responseSerializationInternal{} + responseJSON.StatusCode = resp.StatusCode + responseJSON.Headers = headersAny + responseJSON.HeaderFlat = headerFlat + // Build cookies map from standard bag if len(resp.Cookies) > 0 { responseJSON.Cookies = make(map[string]Cookie) @@ -282,11 +314,12 @@ func (response Response) ToGenericJSON() (any, error) { } } - responseJSON := ResponseSerialization{ - StatusCode: response.StatusCode, - Headers: response.Headers, - HeaderFlat: response.HeaderFlat, - } + responseJSON := responseSerializationInternal{} + + responseJSON.StatusCode = response.StatusCode + responseJSON.Headers = response.Headers + responseJSON.HeaderFlat = response.HeaderFlat + responseJSON.BodyControl = response.BodyControl // Build cookies map from standard bag if len(response.Cookies) > 0 { diff --git a/pkg/lib/api/response_test.go b/pkg/lib/api/response_test.go index 8767e674..09ed96ac 100644 --- a/pkg/lib/api/response_test.go +++ b/pkg/lib/api/response_test.go @@ -73,7 +73,7 @@ func TestResponse_NewResponseFromSpec_StatusCode_not_set(t *testing.T) { } func TestResponse_NewResponse(t *testing.T) { - response, err := NewResponse(200, nil, nil, nil, strings.NewReader("foo"), ResponseFormat{}) + response, err := NewResponse(200, nil, nil, strings.NewReader("foo"), nil, ResponseFormat{}) go_test_utils.ExpectNoError(t, err, "unexpected error") go_test_utils.AssertIntEquals(t, response.StatusCode, 200) } @@ -86,7 +86,7 @@ func TestResponse_String(t *testing.T) { } }` - response, err := NewResponse(200, nil, nil, nil, strings.NewReader(requestString), ResponseFormat{}) + response, err := NewResponse(200, nil, nil, strings.NewReader(requestString), nil, ResponseFormat{}) go_test_utils.ExpectNoError(t, err, "error constructing response") assertString := "200\n\n\n" + requestString @@ -119,7 +119,7 @@ func TestResponse_Cookies(t *testing.T) { if err != nil { t.Fatal(err) } - response, err := NewResponse(res.StatusCode, header, nil, res.Cookies(), res.Body, ResponseFormat{}) + response, err := NewResponse(res.StatusCode, header, res.Cookies(), res.Body, nil, ResponseFormat{}) if err != nil { t.Fatal(err) } diff --git a/pkg/lib/template/template_loader_test.go b/pkg/lib/template/template_loader_test.go index 485cb9b6..c1edfb08 100644 --- a/pkg/lib/template/template_loader_test.go +++ b/pkg/lib/template/template_loader_test.go @@ -82,7 +82,7 @@ func TestBigIntRender(t *testing.T) { inputNumber := "132132132182323" - resp, _ := api.NewResponse(200, nil, nil, nil, strings.NewReader(fmt.Sprintf(`{"bigINT":%s}`, inputNumber)), api.ResponseFormat{}) + resp, _ := api.NewResponse(200, nil, nil, strings.NewReader(fmt.Sprintf(`{"bigINT":%s}`, inputNumber)), nil, api.ResponseFormat{}) respJson, _ := resp.ServerResponseToJsonString(false) store.SetWithQjson(respJson, map[string]string{"testINT": "body.bigINT"}) @@ -408,13 +408,13 @@ func Test_DataStore_QJson(t *testing.T) { 200, map[string]any{"x-header": []string{"foo", "bar"}}, nil, - nil, strings.NewReader(`{ "flib": [ "flab", "flob" ] }`), + nil, api.ResponseFormat{}, ) store := datastore.NewStore(false) diff --git a/test/datastore/check.json b/test/datastore/check.json index 4cca4797..ee07cfae 100644 --- a/test/datastore/check.json +++ b/test/datastore/check.json @@ -5,6 +5,9 @@ "server_url": "http://localhost:9999", "endpoint": "bounce-json", "method": "POST", + "header": { + "x-henk": "denk" + }, "body": [ {{ datastore -3 | qjson "body" }}, {{ datastore -2 | qjson "body" }}, @@ -13,12 +16,19 @@ }, "response": { "statuscode": 200, + "header": { + "Content-Length": "367" + }, "body": { "body": [ {"some": "data"}, {"some": ["more", "data"]}, {"some": "data"} - ] + ], + "body:control": { + "order_matters": true, + "no_extra": true + } } } } diff --git a/test/response/header/header1.json b/test/response/header/header1.json new file mode 100644 index 00000000..979ab865 --- /dev/null +++ b/test/response/header/header1.json @@ -0,0 +1,54 @@ +{ + "name": "test header", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST", + "header": { + "x-key": "value", + "x-key-array": [ + "value1", + "value2" + ] + }, + "body": { + "test": "torsten" + } + }, + "response": { + "statuscode": 200, + "header": { + "Content-Type": "text/plain; charset=utf-8" + }, + "body": { + "header:control": { + "no_extra": true + }, + "header": { + "Connection": [ + "close" + ], + "Content-Length": [ + "18" + ], + "Content-Type": [ + "application/json" + ], + "X-Key": [ + "value" + ], + "X-Key-Array": [ + "value1", + "value2" + ] + }, + "query_params": {}, + "body:control": { + "no_extra": true + }, + "body": { + "test": "torsten" + } + } + } +} \ No newline at end of file diff --git a/test/response/header/header2.json b/test/response/header/header2.json new file mode 100644 index 00000000..898f1986 --- /dev/null +++ b/test/response/header/header2.json @@ -0,0 +1,16 @@ +{ + "name": "test response header (check for a failing test with a reverse result)", + "reverse_test_result": true, + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header": { + "Content-Length": "foo", + "Content-Type": "bar" + } + } +} \ No newline at end of file diff --git a/test/response/header/header3.json b/test/response/header/header3.json new file mode 100644 index 00000000..5d68ccfb --- /dev/null +++ b/test/response/header/header3.json @@ -0,0 +1,19 @@ +{ + "name": "test response header:control for header values", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header": { + "Content-Length:control": { + "match": "^\\d+$" + }, + "Content-Type:control": { + "match": "^text/plain;.*" + } + } + } +} \ No newline at end of file diff --git a/test/response/header/header4.json b/test/response/header/header4.json new file mode 100644 index 00000000..de16104a --- /dev/null +++ b/test/response/header/header4.json @@ -0,0 +1,20 @@ +{ + "name": "test response header:control for header values (check for a failing test with a reverse result)", + "reverse_test_result": true, + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header": { + "Content-Length:control": { + "match": "foo" + }, + "Content-Type:control": { + "match": "bar" + } + } + } +} \ No newline at end of file diff --git a/test/response/header/header5.json b/test/response/header/header5.json new file mode 100644 index 00000000..d5143b55 --- /dev/null +++ b/test/response/header/header5.json @@ -0,0 +1,16 @@ +{ + "name": "test response header", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST" + }, + "response": { + "statuscode": 200, + "header": { + "Content-Type": [ + "text/plain; charset=utf-8" + ] + } + } +} \ No newline at end of file diff --git a/test/response/header/header6.json b/test/response/header/header6.json new file mode 100644 index 00000000..98868355 --- /dev/null +++ b/test/response/header/header6.json @@ -0,0 +1,43 @@ +{ + "name": "check HTTP header using control, use reverse_test_result", + "request": { + "server_url": "http://localhost:9999", + "endpoint": "bounce-json", + "method": "POST", + "body": [ + "henk 1", + "henk 2", + "henk 3" + ] + }, + "response": { + "statuscode": 200, + "header": { + // test the string format + "Content-Length": "237", + "Content-Type": "text/plain; charset=utf-8" + }, + "body:control": { + "no_extra": true + }, + "body": { + "header": { + "Connection": [ + "close" + ], + "Content-Length": [ + "28" + ], + "Content-Type": [ + "application/json" + ] + }, + "query_params": {}, + "body": [ + "henk 1", + "henk 2", + "henk 3" + ] + } + } +} \ No newline at end of file diff --git a/test/response/header/manifest.json b/test/response/header/manifest.json index 4771746f..dc4b15ea 100644 --- a/test/response/header/manifest.json +++ b/test/response/header/manifest.json @@ -4,81 +4,13 @@ "dir": "../_res", "testmode": false }, - "name": "response header: format header_flat", + "name": "response header: format header", "tests": [ - { - "name": "test response header_flat", - "request": { - "server_url": "http://localhost:9999", - "endpoint": "bounce-json", - "method": "POST" - }, - "response": { - "statuscode": 200, - "header": { - "Content-Type": [ - "text/plain; charset=utf-8" - ] - }, - "header_flat": { - "Content-Type": "text/plain; charset=utf-8" - } - } - }, - { - "name": "test response header_flat (check for a failing test with a reverse result)", - "reverse_test_result": true, - "request": { - "server_url": "http://localhost:9999", - "endpoint": "bounce-json", - "method": "POST" - }, - "response": { - "statuscode": 200, - "header_flat": { - "Content-Length": "foo", - "Content-Type": "bar" - } - } - }, - { - "name": "test response header_flat: control for header values", - "request": { - "server_url": "http://localhost:9999", - "endpoint": "bounce-json", - "method": "POST" - }, - "response": { - "statuscode": 200, - "header_flat": { - "Content-Length:control": { - "match": "^\\d+$" - }, - "Content-Type:control": { - "match": "^text/plain;.*" - } - } - } - }, - { - "name": "test response header_flat: control for header values (check for a failing test with a reverse result)", - "reverse_test_result": true, - "request": { - "server_url": "http://localhost:9999", - "endpoint": "bounce-json", - "method": "POST" - }, - "response": { - "statuscode": 200, - "header_flat": { - "Content-Length:control": { - "match": "foo" - }, - "Content-Type:control": { - "match": "bar" - } - } - } - } + "@header1.json" + ,"@header2.json" + ,"@header3.json" + ,"@header4.json" + ,"@header5.json" + ,"@header6.json" ] } \ No newline at end of file From 634e16ed12e7b033ec616b015dc834f1d9a36e21 Mon Sep 17 00:00:00 2001 From: Martin Rode Date: Fri, 1 Nov 2024 09:44:23 +0100 Subject: [PATCH 5/6] Makefile: removed unnecessary targets, fixed apitest --- .github/workflows/unit-tests.yml | 12 ++-------- Makefile | 38 ++++--------------------------- go.sum | 2 -- test/control/header/manifest.json | 3 --- 4 files changed, 7 insertions(+), 48 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ff413a8a..4c3ed728 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -27,17 +27,9 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: execute unit tests + - name: execute unit tests, build & apitest shell: bash - run: make test - - - name: build executable for test suite - shell: bash - run: make build - - - name: execute apitest - shell: bash - run: make apitest + run: make all - name: Notify slack channel about a failure if: ${{ failure() }} diff --git a/Makefile b/Makefile index 5cb364c4..c88a7ec5 100644 --- a/Makefile +++ b/Makefile @@ -1,44 +1,16 @@ -GOOS ?= linux -GOARCH ?= amd64 -GIT_COMMIT_SHA ?= $(shell git rev-list -1 HEAD) -LD_FLAGS = -ldflags="-X main.buildCommit=${GIT_COMMIT_SHA}" - all: test build apitest -deps: - go mod download github.com/clbanning/mxj - go get ./... - -vet: +test: go vet ./... - -fmt: - go fmt ./... - -test: deps fmt vet go test -race -cover ./... -webtest: - go test -coverprofile=testcoverage.out - go tool cover -html=testcoverage.out - -apitest: +apitest: build ./apitest -c apitest.test.yml --stop-on-fail -d test/ -gox: deps - go get github.com/mitchellh/gox - gox ${LDFLAGS} -parallel=4 -output="./bin/apitest_{{.OS}}_{{.Arch}}" - clean: rm -rfv ./apitest ./bin/* ./testcoverage.out -ci: deps - go build $(LD_FLAGS) -o bin/apitest_$(GOOS)_$(GOARCH) *.go - -build: deps - go build $(LD_FLAGS) - -build-linux: deps - GOOS=linux GOARCH=amd64 go build -o apitest-linux +build: + go build -.PHONY: all test apitest webtest gox build clean +.PHONY: all test apitest build clean diff --git a/go.sum b/go.sum index 70957968..8aa4ca27 100644 --- a/go.sum +++ b/go.sum @@ -217,8 +217,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/programmfabrik/go-test-utils v0.0.0-20191114143449-b8e16b04adb1 h1:NbjvVAvjVfIse7/zgFpP0P93Hj3o4lRJCzGgkNQ87Gc= github.com/programmfabrik/go-test-utils v0.0.0-20191114143449-b8e16b04adb1/go.mod h1:6Tg7G+t9KYiFa0sU8PpISt9RUgIpgrEI+tXvWz3tSIU= -github.com/programmfabrik/golib v0.0.0-20240226091422-733aede66819 h1:lJ+a0MLo4Dn2UTF0Q/nh9msLqP8MaNEL/RbJLop022g= -github.com/programmfabrik/golib v0.0.0-20240226091422-733aede66819/go.mod h1:qb4pSUhPsZ/UfvM/MBNwKHb6W7xL85uSi4od9emNHHw= github.com/programmfabrik/golib v0.0.0-20240701125551-843bc5e3be55 h1:VBYGpSvjwHSa5ARrs6uPlUOJF1+n6rFWn49+++h20IU= github.com/programmfabrik/golib v0.0.0-20240701125551-843bc5e3be55/go.mod h1:qb4pSUhPsZ/UfvM/MBNwKHb6W7xL85uSi4od9emNHHw= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= diff --git a/test/control/header/manifest.json b/test/control/header/manifest.json index a612b377..cad3d431 100644 --- a/test/control/header/manifest.json +++ b/test/control/header/manifest.json @@ -20,9 +20,6 @@ "response": { // check actual HTTP headers "header": { - "Content-Type:control": { - "element_count": 1 - }, "xxx:control": { "must_not_exist": true } From 031169bbcf2804757e1326592dd74c7d455b98b0 Mon Sep 17 00:00:00 2001 From: Philipp Hempel Date: Fri, 1 Nov 2024 14:20:07 +0100 Subject: [PATCH 6/6] Readme: added paragraph about body:control; see #73980 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 10fe8e12..98f4a5c6 100644 --- a/README.md +++ b/README.md @@ -817,6 +817,9 @@ In the example we use the jsonObject `test` and define some control structures o } ``` +### `body:control` + +All controls, which are defined below, can also be applied to the complete response body itself by setting `body:control`. The control check functions work the same as on any other key. This can be combined with other controls inside the body. ## Available controls