diff --git a/Makefile b/Makefile index b4223a0cf..2780d02fc 100644 --- a/Makefile +++ b/Makefile @@ -94,10 +94,6 @@ coverage-html: test coverage-qe: build-tnf-tool ./tnf generate qe-coverage-report -# Generates the test catalog in JSON -build-catalog-json: build-tnf-tool - ./tnf generate catalog json >catalog.json - # Generates the test catalog in Markdown build-catalog-md: build-tnf-tool classification-js ./tnf generate catalog markdown >CATALOG.md diff --git a/cmd/tnf/addclaim/addclaim.go b/cmd/tnf/claim/add/add.go similarity index 74% rename from cmd/tnf/addclaim/addclaim.go rename to cmd/tnf/claim/add/add.go index 305c529d9..0d65300a3 100644 --- a/cmd/tnf/addclaim/addclaim.go +++ b/cmd/tnf/claim/add/add.go @@ -1,4 +1,4 @@ -package claim +package add import ( "fmt" @@ -16,24 +16,12 @@ import ( var ( Reportdir string Claim string - Claim1 string - Claim2 string - addclaim = &cobra.Command{ - Use: "claim", - Short: "The test suite generates a \"claim\" file", - RunE: claimUpdate, - } claimAddFile = &cobra.Command{ Use: "add", - Short: "The test suite generates a \"claim\" file", + Short: "Add results from xml junit files to an existing claim file.", RunE: claimUpdate, } - claimCompareFiles = &cobra.Command{ - Use: "compare", - Short: "Compare 2 \"claim\" file", - RunE: claimCompare, - } ) const ( @@ -114,23 +102,6 @@ func NewCommand() *cobra.Command { if err != nil { return nil } - addclaim.AddCommand(claimAddFile) - claimCompareFiles.Flags().StringVarP( - &Claim1, "claim1", "1", "", - "existing claim1 file. (Required) first file to compare", - ) - claimCompareFiles.Flags().StringVarP( - &Claim2, "claim2", "2", "", - "existing claim2 file. (Required) second file to compare with", - ) - err = claimCompareFiles.MarkFlagRequired("claim1") - if err != nil { - return nil - } - err = claimCompareFiles.MarkFlagRequired("claim2") - if err != nil { - return nil - } - addclaim.AddCommand(claimCompareFiles) - return addclaim + + return claimAddFile } diff --git a/cmd/tnf/addclaim/addclaim_test.go b/cmd/tnf/claim/add/add_test.go similarity index 89% rename from cmd/tnf/addclaim/addclaim_test.go rename to cmd/tnf/claim/add/add_test.go index c3313a4f2..4ef364631 100644 --- a/cmd/tnf/addclaim/addclaim_test.go +++ b/cmd/tnf/claim/add/add_test.go @@ -1,4 +1,4 @@ -package claim +package add import ( "testing" @@ -44,6 +44,6 @@ func TestNewCommand(t *testing.T) { // No parameters to test result := NewCommand() assert.NotNil(t, result) - assert.Equal(t, "claim", result.Use) - assert.Equal(t, "The test suite generates a \"claim\" file", result.Short) + assert.Equal(t, "add", result.Use) + assert.Equal(t, "Add results from xml junit files to an existing claim file.", result.Short) } diff --git a/cmd/tnf/claim/claim.go b/cmd/tnf/claim/claim.go new file mode 100644 index 000000000..530d84509 --- /dev/null +++ b/cmd/tnf/claim/claim.go @@ -0,0 +1,23 @@ +package claim + +import ( + "github.com/spf13/cobra" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/add" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/compare" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/show" +) + +var ( + claimCommand = &cobra.Command{ + Use: "claim", + Short: "Help tools for working with claim files.", + } +) + +func NewCommand() *cobra.Command { + claimCommand.AddCommand(add.NewCommand()) + claimCommand.AddCommand(compare.NewCommand()) + claimCommand.AddCommand(show.NewCommand()) + + return claimCommand +} diff --git a/cmd/tnf/addclaim/calim_test3.json b/cmd/tnf/claim/compare/calim_test3.json similarity index 100% rename from cmd/tnf/addclaim/calim_test3.json rename to cmd/tnf/claim/compare/calim_test3.json diff --git a/cmd/tnf/addclaim/claim_test1.json b/cmd/tnf/claim/compare/claim_test1.json similarity index 100% rename from cmd/tnf/addclaim/claim_test1.json rename to cmd/tnf/claim/compare/claim_test1.json diff --git a/cmd/tnf/addclaim/claim_test2.json b/cmd/tnf/claim/compare/claim_test2.json similarity index 100% rename from cmd/tnf/addclaim/claim_test2.json rename to cmd/tnf/claim/compare/claim_test2.json diff --git a/cmd/tnf/addclaim/compareclaim.go b/cmd/tnf/claim/compare/compare.go similarity index 84% rename from cmd/tnf/addclaim/compareclaim.go rename to cmd/tnf/claim/compare/compare.go index 7f22d0d05..160f6ce05 100644 --- a/cmd/tnf/addclaim/compareclaim.go +++ b/cmd/tnf/claim/compare/compare.go @@ -1,4 +1,4 @@ -package claim +package compare import ( "encoding/json" @@ -7,30 +7,41 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim" ) -type claimFileStruct struct { - Claim struct { - Nodes struct { - CniPlugins map[string][]Cni `json:"cniPlugins"` - NodesHwInfo map[string]interface{} `json:"nodesHwInfo"` - CsiDriver interface{} `json:"csiDriver"` - } `json:"nodes"` - - RawResults struct { - Cnfcertificationtest struct { - Testsuites struct { - Testsuite struct { - Testcase []testCase `json:"testcase"` - } `json:"testsuite"` - } `json:"testsuites"` - } `json:"cnf-certification-test"` - } `json:"rawResults"` - } `json:"claim"` -} -type Cni struct { - Name string "json:\"name\"" - Plugins []interface{} "json:\"plugins\"" +var ( + Claim1 string + Claim2 string + + claimCompareFiles = &cobra.Command{ + Use: "compare", + Short: "Compare two claim files.", + RunE: claimCompare, + } +) + +func NewCommand() *cobra.Command { + claimCompareFiles.Flags().StringVarP( + &Claim1, "claim1", "1", "", + "existing claim1 file. (Required) first file to compare", + ) + claimCompareFiles.Flags().StringVarP( + &Claim2, "claim2", "2", "", + "existing claim2 file. (Required) second file to compare", + ) + err := claimCompareFiles.MarkFlagRequired("claim1") + if err != nil { + log.Errorf("Failed to mark flag claim1 as required: %v", err) + return nil + } + err = claimCompareFiles.MarkFlagRequired("claim2") + if err != nil { + log.Errorf("Failed to mark flag claim2 as required: %v", err) + return nil + } + + return claimCompareFiles } func claimCompare(_ *cobra.Command, _ []string) error { @@ -43,11 +54,6 @@ func claimCompare(_ *cobra.Command, _ []string) error { return nil } -type testCase struct { - Name string `json:"-name"` - Status string `json:"-status"` -} - func claimCompareFilesfunc(claim1, claim2 string) error { // readfiles claimdata1, err := os.ReadFile(claim1) @@ -87,8 +93,8 @@ func claimCompareFilesfunc(claim1, claim2 string) error { return nil } -func unmarshalClaimFile(claimdata []byte) (claimFileStruct, error) { - var claimDataResult claimFileStruct +func unmarshalClaimFile(claimdata []byte) (claim.Schema, error) { + var claimDataResult claim.Schema errclaimDataResult := json.Unmarshal(claimdata, &claimDataResult) if errclaimDataResult != nil { log.Fatalf("Error in unmarshal the claim file :%v", errclaimDataResult) @@ -127,11 +133,11 @@ func compareEqual2String(a, b []string) bool { return true } -// compare between 2 test case result (testCase) object +// compare between 2 test case result (claim.TestCase) object // return 3 values: 1. the test name that have different result value - diffResult // 2. name of test cases that in claim2 but do not have them on claim1 - notFoundtestIn1 // 2. name of test cases that in claim1 but do not have them on claim2 - notFoundtestIn2 -func compare2TestCaseResults(testcaseResult1, testcaseResult2 []testCase) (diffResult []testCase, notFoundtestIn1, notFoundtestIn2 []string) { +func compare2TestCaseResults(testcaseResult1, testcaseResult2 []claim.TestCaseRawResult) (diffResult []claim.TestCaseRawResult, notFoundtestIn1, notFoundtestIn2 []string) { var testcaseR1, testcaseR2 []string for _, result1 := range testcaseResult1 { testcaseR1 = append(testcaseR1, result1.Name) @@ -195,7 +201,7 @@ func removeDuplicateValues(intSlice []string) []string { } // compare between 2 cni objects and print the difference -func compare2cni(cni1, cni2 map[string][]Cni) { +func compare2cni(cni1, cni2 map[string][]claim.Cni) { for node, val := range cni1 { for node2, val2 := range cni2 { if node != node2 { @@ -220,7 +226,7 @@ func compare2cni(cni1, cni2 map[string][]Cni) { // 1. name of cni's that have same name but the plugin value are different - diffPlugins // 2. name of cni's that found on claim2 but not in claim1 - notFoundNamesIn1 // 3. name of cni's that found on claim1 but not in claim2 - notFoundNamesIn3 -func compare2cniHelper(cniList1, cniList2 []Cni, node string) (diffPlugins []Cni, notFoundNamesIn1, notFoundNamesIn2 []string) { +func compare2cniHelper(cniList1, cniList2 []claim.Cni, node string) (diffPlugins []claim.Cni, notFoundNamesIn1, notFoundNamesIn2 []string) { var cniList1Name, cniList2Name []string if len(cniList1) == 0 { log.Infof("in node %s CNIs present in claim2 and on claim1 that node do not have cni values: %v", node, cniList2) diff --git a/cmd/tnf/addclaim/compareclaim_test.go b/cmd/tnf/claim/compare/compare_test.go similarity index 87% rename from cmd/tnf/addclaim/compareclaim_test.go rename to cmd/tnf/claim/compare/compare_test.go index 4b954e765..3db1cf58a 100644 --- a/cmd/tnf/addclaim/compareclaim_test.go +++ b/cmd/tnf/claim/compare/compare_test.go @@ -1,39 +1,41 @@ -package claim +package compare import ( "reflect" "testing" + + "github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim" ) func Test_compare2TestCaseResults(t *testing.T) { type args struct { - testcaseResult1 []testCase - testcaseResult2 []testCase + testcaseResult1 []claim.TestCaseRawResult + testcaseResult2 []claim.TestCaseRawResult } tests := []struct { name string args args - wantDiffresult []testCase + wantDiffresult []claim.TestCaseRawResult wantNotFoundtest []string wantNotFoundtest2 []string }{ { name: "test1", args: args{ - testcaseResult1: []testCase{ + testcaseResult1: []claim.TestCaseRawResult{ { Name: "[It] observability observability-container-logging [common, observability, observability-container-logging]", Status: "skipped", }, }, - testcaseResult2: []testCase{ + testcaseResult2: []claim.TestCaseRawResult{ { Name: "[It] observability observability-container-logging [common, observability, observability-container-logging]", Status: "failed", }, }, }, - wantDiffresult: []testCase{ + wantDiffresult: []claim.TestCaseRawResult{ { Name: "[It] observability observability-container-logging [common, observability, observability-container-logging]", Status: "skipped", @@ -45,7 +47,7 @@ func Test_compare2TestCaseResults(t *testing.T) { { name: "test2", args: args{ - testcaseResult1: []testCase{ + testcaseResult1: []claim.TestCaseRawResult{ { Name: "[It] observability observability-crd-status [common, observability, observability-crd-status]", Status: "skipped", @@ -55,14 +57,14 @@ func Test_compare2TestCaseResults(t *testing.T) { Status: "skipped", }, }, - testcaseResult2: []testCase{ + testcaseResult2: []claim.TestCaseRawResult{ { Name: "[It] observability observability-container-logging [common, observability, observability-container-logging]", Status: "failed", }, }, }, - wantDiffresult: []testCase{ + wantDiffresult: []claim.TestCaseRawResult{ { Name: "[It] observability observability-container-logging [common, observability, observability-container-logging]", Status: "skipped", @@ -90,21 +92,21 @@ func Test_compare2TestCaseResults(t *testing.T) { func Test_compare2cnis(t *testing.T) { type args struct { - cniList1 []Cni - cniList2 []Cni + cniList1 []claim.Cni + cniList2 []claim.Cni nodeName string } tests := []struct { name string args args - wantDiffplugins []Cni + wantDiffplugins []claim.Cni wantNotFoundNames []string wantNotFoundNames2 []string }{ { name: "test1", args: args{ - cniList1: []Cni{ + cniList1: []claim.Cni{ { Name: "podman", Plugins: nil, @@ -114,7 +116,7 @@ func Test_compare2cnis(t *testing.T) { Plugins: nil, }, }, - cniList2: []Cni{ + cniList2: []claim.Cni{ { Name: "podman", Plugins: nil, @@ -129,13 +131,13 @@ func Test_compare2cnis(t *testing.T) { { name: "test2", args: args{ - cniList1: []Cni{ + cniList1: []claim.Cni{ { Name: "podman", Plugins: nil, }, }, - cniList2: []Cni{ + cniList2: []claim.Cni{ { Name: "podman", Plugins: nil, @@ -155,7 +157,7 @@ func Test_compare2cnis(t *testing.T) { name: "test3", args: args{ cniList1: nil, - cniList2: []Cni{ + cniList2: []claim.Cni{ { Name: "podman", Plugins: nil, @@ -185,8 +187,8 @@ func Test_compare2cnis(t *testing.T) { { name: "test5", args: args{ - cniList1: []Cni{}, - cniList2: []Cni{}, + cniList1: []claim.Cni{}, + cniList2: []claim.Cni{}, nodeName: "master1", }, wantDiffplugins: nil, @@ -196,13 +198,13 @@ func Test_compare2cnis(t *testing.T) { { name: "test6", args: args{ - cniList1: []Cni{ + cniList1: []claim.Cni{ { Name: "podman", Plugins: nil, }, }, - cniList2: []Cni{}, + cniList2: []claim.Cni{}, nodeName: "master1", }, wantDiffplugins: nil, diff --git a/cmd/tnf/claim/show/failures/failures.go b/cmd/tnf/claim/show/failures/failures.go new file mode 100644 index 000000000..afe2d4876 --- /dev/null +++ b/cmd/tnf/claim/show/failures/failures.go @@ -0,0 +1,304 @@ +package failures + +import ( + "encoding/json" + "fmt" + "log" + "strings" + + "github.com/spf13/cobra" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim" + "github.com/test-network-function/cnf-certification-test/pkg/testhelper" +) + +var ( + claimFilePathFlag string + testSuitesFlag string + outputFormatFlag string + + showFailuresCommand = &cobra.Command{ + Use: "failures", + Short: "Shows failed test cases from a claim file.", + Long: `Parses a claim.json file and shows a report containing only the failed test cases per test suite. +For each failed test case, shows every non compliant object in a readable way in order to help users to understand the +failure reasons. Using the flag "--output json", the program will print a json representation of those failed test cases. +A comma separated list of test suites can be provided with the flag "--testsuites "testSuite1,testSuite2", so the +output will only print failed test cases from those test suites only. +`, + Example: `./tnf claim show failures --claim path/to/claim.json +Test Suite: access-control + Test Case: access-control-sys-admin-capability-check + Description: Ensures that containers do not use SYS_ADMIN capability + Failure reasons: + 1 - Type: Container, Reason: Non compliant capability detected in container + Namespace: tnf, Pod Name: test-887998557-8gwwm, Container Name: test, SCC Capability: SYS_ADMIN + 2 - Type: Container, Reason: Non compliant capability detected in container + Namespace: tnf, Pod Name: test-887998557-pr2w5, Container Name: test, SCC Capability: SYS_ADMIN + Test Case: access-control-security-context + Description: Checks the security context matches one of the 4 categories + Failure reasons: + 1 - Type: ContainerCategory, Reason: container category is NOT category 1 or category NoUID0 + Namespace: tnf, Pod Name: jack-6f88b5bfb4-q5cw6, Container Name: jack, Category: CategoryID4(anything not matching lower category) + 2 - ... + ... +Test Suite: lifecycle + Test Case: ... + ... + +./tnf claim show failures --claim path/to/claim.json --o json +{ + "testSuites": [ + { + "name": "access-control", + "failures": [ + { + "name": "access-control-sys-admin-capability-check", + "description": "Ensures that containers do not use SYS_ADMIN capability", + "nonCompliantObjects": [ + { + "type": "Container", + "reason": "Non compliant capability detected in container", + "spec": { + "Namespace": "tnf", + "Pod Name": "test-887998557-8gwwm", + "Container Name": "test", + "SCC Capability": "SYS_ADMIN" + } + }, + { + "type": "Container", + "reason": "Non compliant capability detected in container", + "spec": { + "Namespace": "tnf", + "Pod Name": "test-887998557-pr2w5", + "Container Name": "test", + "SCC Capability": "SYS_ADMIN" + } + } + ] + }, + ] + }, + { + "name" : "lifecycle", + "failures": [ + ... + ... + ] + }, + ] + } +`, + RunE: showFailures, + } +) + +const ( + outputFormatText = "text" + outputFormatJSON = "json" + outputFarmatInvalid = "invalid" +) + +var availableOutputFormats = []string{ + outputFormatText, outputFormatJSON, +} + +func NewCommand() *cobra.Command { + showFailuresCommand.Flags().StringVarP(&claimFilePathFlag, "claim", "c", "", + "Required: Existing claim file path.", + ) + + err := showFailuresCommand.MarkFlagRequired("claim") + if err != nil { + log.Fatalf("Failed to mark claim file path as required parameter: %v", err) + return nil + } + + // This command accepts a (optional) list of comma separated suite to filter the + // output. Only the failures from those test suites will be printed. + showFailuresCommand.Flags().StringVarP(&testSuitesFlag, "testsuites", "s", "", + "Optional: comma separated list of test suites names whose failures will be shown.", + ) + + // The format of the output can be changed. Default is plain text, but it can also print + // it in json format. + showFailuresCommand.Flags().StringVarP(&outputFormatFlag, "output", "o", outputFormatText, + fmt.Sprintf("Optional: output format. Available formats: %v", availableOutputFormats), + ) + + return showFailuresCommand +} + +// Parses the comma separated list to create a helper map, whose +// keys are the test suite names. +func parseTargetTestSuitesFlag() map[string]bool { + if testSuitesFlag == "" { + return nil + } + + targetTestSuites := map[string]bool{} + for _, testSuite := range strings.Split(testSuitesFlag, ",") { + targetTestSuites[strings.TrimSpace(testSuite)] = true + } + + return targetTestSuites +} + +// Parses the output format flag. Returns error if the format +// does not appear in the list "availableOutputFormats". +func parseOutputFormatFlag() (string, error) { + for _, outputFormat := range availableOutputFormats { + if outputFormat == outputFormatFlag { + return outputFormat, nil + } + } + + return "", fmt.Errorf("invalid output format flag %q - available formats: %v", outputFormatFlag, availableOutputFormats) +} + +// Parses the claim's test case's failureReason field and creates a list +// of NonCompliantObject's. +func getNonCompliantObjectsFromFailureReason(failureReason string) ([]NonCompliantObject, error) { + objects := struct { + Compliant []testhelper.ReportObject `json:"CompliantObjectsOut"` + NonCompliant []testhelper.ReportObject `json:"NonCompliantObjectsOut"` + }{} + + err := json.Unmarshal([]byte(failureReason), &objects) + if err != nil { + return nil, fmt.Errorf("failed to decode failureReason %s: %v", failureReason, err) + } + + // Now let's create a list of our NonCompliantObject-type items. + nonCompliantObjects := []NonCompliantObject{} + for _, object := range objects.NonCompliant { + outputObject := NonCompliantObject{Type: object.ObjectType, Reason: object.ObjectFieldsValues[0]} + for i := 1; i < len(object.ObjectFieldsKeys); i++ { + outputObject.Spec.AddField(object.ObjectFieldsKeys[i], object.ObjectFieldsValues[i]) + } + + nonCompliantObjects = append(nonCompliantObjects, outputObject) + } + + return nonCompliantObjects, nil +} + +// Prints the failures in plain text. +func printFailuresText(testSuites []FailedTestSuite) { + for _, ts := range testSuites { + fmt.Printf("Test Suite: %s\n", ts.TestSuiteName) + for _, tc := range ts.FailingTestCases { + fmt.Printf(" Test Case: %s\n", tc.TestCaseName) + fmt.Printf(" Description: %s\n", tc.TestCaseDescription) + fmt.Printf(" Failure reasons:\n") + for i := range tc.NonCompliantObjects { + nonCompliantObject := tc.NonCompliantObjects[i] + fmt.Printf(" %2d - Type: %s, Reason: %s\n", i+1, nonCompliantObject.Type, nonCompliantObject.Reason) + fmt.Printf(" ") + for i := range nonCompliantObject.Spec.fields { + if i != 0 { + fmt.Printf(", ") + } + field := nonCompliantObject.Spec.fields[i] + fmt.Printf("%s: %s", field.key, field.value) + } + fmt.Printf("\n") + } + } + } +} + +// Prints the failures in json format. +func printFailuresJSON(testSuites []FailedTestSuite) { + type ClaimFailures struct { + Failures []FailedTestSuite `json:"testSuites"` + } + + claimFailures := ClaimFailures{Failures: testSuites} + bytes, err := json.MarshalIndent(claimFailures, "", " ") + if err != nil { + log.Fatalf("failed to marshal failures: %v", err) + } + + fmt.Printf("%s\n", string(bytes)) +} + +// Creates a list of FailingTestSuite from the results parsed from a claim file. The parsed +// results in claimResultsByTestSuite var maps a test suite name to a list of TestCaseResult, +// which are processed to create the list of FailingTestSuite, filtering out those test suites +// that don't exist in the targetTestSuites map. +func getFailedTestCasesByTestSuite(claimResultsByTestSuite map[string][]*claim.TestCaseResult, targetTestSuites map[string]bool) ([]FailedTestSuite, error) { + testSuites := []FailedTestSuite{} + for testSuite := range claimResultsByTestSuite { + if targetTestSuites != nil && !targetTestSuites[testSuite] { + continue + } + + failedTcs := []FailedTestCase{} + for _, tc := range claimResultsByTestSuite[testSuite] { + if tc.State != "failed" { + continue + } + + nonCompliantObjects, err := getNonCompliantObjectsFromFailureReason(tc.FailureReason) + if err != nil { + return nil, fmt.Errorf("test suite %s, test case %s : failed to parse non compliant objects: %v", testSuite, tc.TestID.ID, err) + } + + failingTc := FailedTestCase{ + TestCaseName: tc.TestID.ID, + TestCaseDescription: tc.Description, + NonCompliantObjects: nonCompliantObjects, + } + + failedTcs = append(failedTcs, failingTc) + } + + if len(failedTcs) > 0 { + testSuites = append(testSuites, FailedTestSuite{ + TestSuiteName: testSuite, + FailingTestCases: failedTcs, + }) + } + } + + return testSuites, nil +} + +// Main function for the `show failures` subcommand. +func showFailures(_ *cobra.Command, _ []string) error { + outputFormat, err := parseOutputFormatFlag() + if err != nil { + return err + } + + // Parse the claim file into the claim scheme. + claimScheme, err := claim.Parse(claimFilePathFlag) + if err != nil { + return fmt.Errorf("failed to parse claim file %s: %v", claimFilePathFlag, err) + } + + // Order test case results by test suite, using a helper map. + resultsByTestSuite := map[string][]*claim.TestCaseResult{} + for id := range claimScheme.Claim.Results { + tcResult := claimScheme.Claim.Results[id][0] + resultsByTestSuite[tcResult.TestID.Suite] = append(resultsByTestSuite[tcResult.TestID.Suite], &tcResult) + } + + targetTestSuites := parseTargetTestSuitesFlag() + // From the target test suites, get their failed test cases and put them in + // our custom types. + testSuites, err := getFailedTestCasesByTestSuite(resultsByTestSuite, targetTestSuites) + if err != nil { + return fmt.Errorf("failed to process claim file results: %v", err) + } + + switch outputFormat { + case outputFormatJSON: + printFailuresJSON(testSuites) + default: + printFailuresText(testSuites) + } + + return nil +} diff --git a/cmd/tnf/claim/show/failures/types.go b/cmd/tnf/claim/show/failures/types.go new file mode 100644 index 000000000..130f29601 --- /dev/null +++ b/cmd/tnf/claim/show/failures/types.go @@ -0,0 +1,51 @@ +package failures + +import "fmt" + +// Custom object type needed to provide a different JSON serialization than +// the one in claim's test cases' failureReason field. +type NonCompliantObject struct { + Type string `json:"type"` + Reason string `json:"reason"` + Spec ObjectSpec `json:"spec"` +} + +type ObjectSpec struct { + fields []struct{ key, value string } +} + +func (spec *ObjectSpec) AddField(key, value string) { + spec.fields = append(spec.fields, struct { + key string + value string + }{key, value}) +} + +func (spec *ObjectSpec) MarshalJSON() ([]byte, error) { + if len(spec.fields) == 0 { + return []byte("{}"), nil + } + + specStr := "{" + for i := range spec.fields { + if i != 0 { + specStr += ", " + } + specStr += fmt.Sprintf("%q:%q", spec.fields[i].key, spec.fields[i].value) + } + + specStr += "}" + + return []byte(specStr), nil +} + +type FailedTestCase struct { + TestCaseName string `json:"name"` + TestCaseDescription string `json:"description"` + NonCompliantObjects []NonCompliantObject `json:"nonCompliantObjects"` +} + +type FailedTestSuite struct { + TestSuiteName string `json:"name"` + FailingTestCases []FailedTestCase `json:"failures"` +} diff --git a/cmd/tnf/claim/show/show.go b/cmd/tnf/claim/show/show.go new file mode 100644 index 000000000..0237adee0 --- /dev/null +++ b/cmd/tnf/claim/show/show.go @@ -0,0 +1,19 @@ +package show + +import ( + "github.com/spf13/cobra" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/show/failures" +) + +var ( + showCommand = &cobra.Command{ + Use: "show", + Short: "Shows information from a claim file.", + } +) + +func NewCommand() *cobra.Command { + showCommand.AddCommand(failures.NewCommand()) + + return showCommand +} diff --git a/cmd/tnf/generate/catalog/catalog.go b/cmd/tnf/generate/catalog/catalog.go index d08534b73..78417a092 100644 --- a/cmd/tnf/generate/catalog/catalog.go +++ b/cmd/tnf/generate/catalog/catalog.go @@ -36,12 +36,12 @@ var ( // generateCmd is the root of the "catalog generate" CLI program. generateCmd = &cobra.Command{ Use: "catalog", - Short: "Generates the test catalog", + Short: "Generates the test catalog.", } markdownGenerateClassification = &cobra.Command{ Use: "javascript", - Short: "Generates java script file for classification", + Short: "Generates java script file for classification.", RunE: generateJS, } diff --git a/cmd/tnf/generate/feedback/feedback.go b/cmd/tnf/generate/feedback/feedback.go index fccab6956..1b17c58a2 100644 --- a/cmd/tnf/generate/feedback/feedback.go +++ b/cmd/tnf/generate/feedback/feedback.go @@ -33,7 +33,7 @@ var ( // generateCmd is the root of the "catalog generate" CLI program. generateFeedbackJsFile = &cobra.Command{ Use: "feedbackjs", - Short: "Generates a javascript file called feedback.js from a feedback.json that was downloaded from the results html viewer", + Short: "Generates a javascript file called feedback.js from a feedback.json that was downloaded from the results html viewer.", RunE: runGenerateFeedbackJsFile, } ) diff --git a/cmd/tnf/generate/generate.go b/cmd/tnf/generate/generate.go new file mode 100644 index 000000000..20cc84a23 --- /dev/null +++ b/cmd/tnf/generate/generate.go @@ -0,0 +1,23 @@ +package generate + +import ( + "github.com/spf13/cobra" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/catalog" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/feedback" + qecoverage "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/qe_coverage" +) + +var ( + generate = &cobra.Command{ + Use: "generate", + Short: "generator tool for various tnf artifacts.", + } +) + +func NewCommand() *cobra.Command { + generate.AddCommand(catalog.NewCommand()) + generate.AddCommand(feedback.NewCommand()) + generate.AddCommand(qecoverage.NewCommand()) + + return generate +} diff --git a/cmd/tnf/generate/qe_coverage/qe_coverage.go b/cmd/tnf/generate/qe_coverage/qe_coverage.go index 14e7d396f..a4ee1656f 100644 --- a/cmd/tnf/generate/qe_coverage/qe_coverage.go +++ b/cmd/tnf/generate/qe_coverage/qe_coverage.go @@ -24,9 +24,15 @@ type TestSuiteQeCoverage struct { NotImplementedTestCases []string } +func NewCommand() *cobra.Command { + qeCoverageReportCmd.PersistentFlags().String("suitename", "", "Displays the remaining tests not covered by QE for the specified suite name.") + + return qeCoverageReportCmd +} + var ( // QeCoverageReportCmd is used to generate a QE coverage report. - QeCoverageReportCmd = &cobra.Command{ + qeCoverageReportCmd = &cobra.Command{ Use: "qe-coverage-report", Short: "Generates the current QE coverage report.", Run: func(cmd *cobra.Command, args []string) { diff --git a/cmd/tnf/main.go b/cmd/tnf/main.go index 0bb96a75f..0f6cc5c86 100644 --- a/cmd/tnf/main.go +++ b/cmd/tnf/main.go @@ -4,31 +4,20 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - claim "github.com/test-network-function/cnf-certification-test/cmd/tnf/addclaim" - "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/catalog" - feedback "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/feedback" - qecoverage "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate/qe_coverage" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/claim" + "github.com/test-network-function/cnf-certification-test/cmd/tnf/generate" ) var ( rootCmd = &cobra.Command{ Use: "tnf", - Short: "A CLI for creating, validating, and test-network-function tests.", - } - - generate = &cobra.Command{ - Use: "generate", - Short: "generator tool for various tnf artifacts.", + Short: "A CLI program with tools related to the CNF Certification Suite.", } ) func main() { rootCmd.AddCommand(claim.NewCommand()) - rootCmd.AddCommand(generate) - generate.AddCommand(catalog.NewCommand()) - generate.AddCommand(feedback.NewCommand()) - generate.AddCommand(qecoverage.QeCoverageReportCmd) - qecoverage.QeCoverageReportCmd.PersistentFlags().String("suitename", "", "Displays the remaining tests not covered by QE for the specified suite name") + rootCmd.AddCommand(generate.NewCommand()) if err := rootCmd.Execute(); err != nil { log.Fatal(err) diff --git a/cmd/tnf/pkg/claim/claim.go b/cmd/tnf/pkg/claim/claim.go new file mode 100644 index 000000000..2b75a7171 --- /dev/null +++ b/cmd/tnf/pkg/claim/claim.go @@ -0,0 +1,70 @@ +package claim + +import ( + "encoding/json" + "fmt" + "os" +) + +type Cni struct { + Name string "json:\"name\"" + Plugins []interface{} "json:\"plugins\"" +} + +type TestCaseRawResult struct { + Name string `json:"-name"` + Status string `json:"-status"` +} + +type TestCaseResult struct { + TestID struct { + ID string `json:"id"` + Suite string `json:"suite"` + Tags string `json:"tags"` + } + Description string `json:"testText"` + + Output string `json:"CapturedTestOutput"` + FailureReason string `json:"failureReason"` + State string `json:"state"` + + StartTime string `json:"startTime"` + EndTime string `json:"endTime"` +} + +type Schema struct { + Claim struct { + Nodes struct { + CniPlugins map[string][]Cni `json:"cniPlugins"` + NodesHwInfo map[string]interface{} `json:"nodesHwInfo"` + CsiDriver interface{} `json:"csiDriver"` + } `json:"nodes"` + + RawResults struct { + Cnfcertificationtest struct { + Testsuites struct { + Testsuite struct { + Testcase []TestCaseRawResult `json:"testcase"` + } `json:"testsuite"` + } `json:"testsuites"` + } `json:"cnf-certification-test"` + } `json:"rawResults"` + + Results map[string][]TestCaseResult `json:"results"` + } `json:"claim"` +} + +func Parse(filePath string) (*Schema, error) { + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failure reading file: %v", err) + } + + claimFile := Schema{} + err = json.Unmarshal(fileBytes, &claimFile) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal file: %v", err) + } + + return &claimFile, nil +}