CLI tool subcommand to show failed test cases from a claim file. (#1249)

* Normalize folders structure & minor refactors.

Moved every command and subcommand to separate folders.

Also, removed the "build-catalog-json:" target from the Makefile as that
subcommand (generate catalog json) doesn't exist since long ago.

* CLI tool subcommand to show failed test cases from a claim file.

Added a new subcommand to the tnf cli tool that parses a claim file and
shows the failed test cases in a more readable way. By default, the
output is plain text, but a 'json' output mode can be selected in order
to get a json representation of those failed test cases per test suite.

Also, the subcommand has a flag --testsuites that can be used to reduce
the output to the tests suites included in the comma separated list.

Examples:
1. Show all the failed test cases from all test suites:
./tnf claim show failures --claim path/to/claim.json

2. Show all the failed test cases from all tests suites in json output.
./tnf claim show failures --claim path/to/claim.json --output json

3. Show all the failed test cases from the lifecycle and access-control
   suites.
./tnf claim show failures --claim path/to/claim.json --testsuites
"lifecycle,access-control"

* Adressing initial comments from Brandon.

* Added another error log trace.

Makefile

@@ -94,10 +94,6 @@ coverage-html: test
 coverage-qe: build-tnf-tool
 	./tnf generate qe-coverage-report
 
-# Generates the test catalog in JSON
-build-catalog-json: build-tnf-tool
-	./tnf generate catalog json >catalog.json
-
 # Generates the test catalog in Markdown
 build-catalog-md: build-tnf-tool classification-js
 	./tnf generate catalog markdown >CATALOG.md

cmd/tnf/addclaim/addclaim.go → cmd/tnf/claim/add/add.go

@@ -1,4 +1,4 @@
-package claim
+package add
 
 import (
 	"fmt"
@@ -16,24 +16,12 @@ import (
 var (
 	Reportdir string
 	Claim     string
-	Claim1    string
-	Claim2    string
 
-	addclaim = &cobra.Command{
-		Use:   "claim",
-		Short: "The test suite generates a \"claim\" file",
-		RunE:  claimUpdate,
-	}
 	claimAddFile = &cobra.Command{
 		Use:   "add",
-		Short: "The test suite generates a \"claim\" file",
+		Short: "Add results from xml junit files to an existing claim file.",
 		RunE:  claimUpdate,
 	}
-	claimCompareFiles = &cobra.Command{
-		Use:   "compare",
-		Short: "Compare 2 \"claim\" file",
-		RunE:  claimCompare,
-	}
 )
 
 const (
@@ -114,23 +102,6 @@ func NewCommand() *cobra.Command {
 	if err != nil {
 		return nil
 	}
-	addclaim.AddCommand(claimAddFile)
-	claimCompareFiles.Flags().StringVarP(
-		&Claim1, "claim1", "1", "",
-		"existing claim1 file. (Required) first file to compare",
-	)
-	claimCompareFiles.Flags().StringVarP(
-		&Claim2, "claim2", "2", "",
-		"existing claim2 file. (Required) second file to compare with",
-	)
-	err = claimCompareFiles.MarkFlagRequired("claim1")
-	if err != nil {
-		return nil
-	}
-	err = claimCompareFiles.MarkFlagRequired("claim2")
-	if err != nil {
-		return nil
-	}
-	addclaim.AddCommand(claimCompareFiles)
-	return addclaim
+
+	return claimAddFile
 }

cmd/tnf/addclaim/addclaim_test.go → cmd/tnf/claim/add/add_test.go

@@ -1,4 +1,4 @@
-package claim
+package add
 
 import (
 	"testing"
@@ -44,6 +44,6 @@ func TestNewCommand(t *testing.T) {
 	// No parameters to test
 	result := NewCommand()
 	assert.NotNil(t, result)
-	assert.Equal(t, "claim", result.Use)
-	assert.Equal(t, "The test suite generates a \"claim\" file", result.Short)
+	assert.Equal(t, "add", result.Use)
+	assert.Equal(t, "Add results from xml junit files to an existing claim file.", result.Short)
 }

cmd/tnf/claim/claim.go

@@ -0,0 +1,23 @@
+package claim
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/add"
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/compare"
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/claim/show"
+)
+
+var (
+	claimCommand = &cobra.Command{
+		Use:   "claim",
+		Short: "Help tools for working with claim files.",
+	}
+)
+
+func NewCommand() *cobra.Command {
+	claimCommand.AddCommand(add.NewCommand())
+	claimCommand.AddCommand(compare.NewCommand())
+	claimCommand.AddCommand(show.NewCommand())
+
+	return claimCommand
+}

cmd/tnf/addclaim/compareclaim.go → cmd/tnf/claim/compare/compare.go

@@ -1,4 +1,4 @@
-package claim
+package compare
 
 import (
 	"encoding/json"
@@ -7,30 +7,41 @@ import (
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim"
 )
 
-type claimFileStruct struct {
-	Claim struct {
-		Nodes struct {
-			CniPlugins  map[string][]Cni       `json:"cniPlugins"`
-			NodesHwInfo map[string]interface{} `json:"nodesHwInfo"`
-			CsiDriver   interface{}            `json:"csiDriver"`
-		} `json:"nodes"`
-
-		RawResults struct {
-			Cnfcertificationtest struct {
-				Testsuites struct {
-					Testsuite struct {
-						Testcase []testCase `json:"testcase"`
-					} `json:"testsuite"`
-				} `json:"testsuites"`
-			} `json:"cnf-certification-test"`
-		} `json:"rawResults"`
-	} `json:"claim"`
-}
-type Cni struct {
-	Name    string        "json:\"name\""
-	Plugins []interface{} "json:\"plugins\""
+var (
+	Claim1 string
+	Claim2 string
+
+	claimCompareFiles = &cobra.Command{
+		Use:   "compare",
+		Short: "Compare two claim files.",
+		RunE:  claimCompare,
+	}
+)
+
+func NewCommand() *cobra.Command {
+	claimCompareFiles.Flags().StringVarP(
+		&Claim1, "claim1", "1", "",
+		"existing claim1 file. (Required) first file to compare",
+	)
+	claimCompareFiles.Flags().StringVarP(
+		&Claim2, "claim2", "2", "",
+		"existing claim2 file. (Required) second file to compare",
+	)
+	err := claimCompareFiles.MarkFlagRequired("claim1")
+	if err != nil {
+		log.Errorf("Failed to mark flag claim1 as required: %v", err)
+		return nil
+	}
+	err = claimCompareFiles.MarkFlagRequired("claim2")
+	if err != nil {
+		log.Errorf("Failed to mark flag claim2 as required: %v", err)
+		return nil
+	}
+
+	return claimCompareFiles
 }
 
 func claimCompare(_ *cobra.Command, _ []string) error {
@@ -43,11 +54,6 @@ func claimCompare(_ *cobra.Command, _ []string) error {
 	return nil
 }
 
-type testCase struct {
-	Name   string `json:"-name"`
-	Status string `json:"-status"`
-}
-
 func claimCompareFilesfunc(claim1, claim2 string) error {
 	// readfiles
 	claimdata1, err := os.ReadFile(claim1)
@@ -87,8 +93,8 @@ func claimCompareFilesfunc(claim1, claim2 string) error {
 	return nil
 }
 
-func unmarshalClaimFile(claimdata []byte) (claimFileStruct, error) {
-	var claimDataResult claimFileStruct
+func unmarshalClaimFile(claimdata []byte) (claim.Schema, error) {
+	var claimDataResult claim.Schema
 	errclaimDataResult := json.Unmarshal(claimdata, &claimDataResult)
 	if errclaimDataResult != nil {
 		log.Fatalf("Error in unmarshal the claim file :%v", errclaimDataResult)
@@ -127,11 +133,11 @@ func compareEqual2String(a, b []string) bool {
 	return true
 }
 
-// compare between 2 test case result (testCase) object
+// compare between 2 test case result (claim.TestCase) object
 // return 3 values: 1. the test name that have different result value - diffResult
 // 2. name of test cases that in claim2 but do not have them on claim1 - notFoundtestIn1
 // 2. name of test cases that in claim1 but do not have them on claim2 - notFoundtestIn2
-func compare2TestCaseResults(testcaseResult1, testcaseResult2 []testCase) (diffResult []testCase, notFoundtestIn1, notFoundtestIn2 []string) {
+func compare2TestCaseResults(testcaseResult1, testcaseResult2 []claim.TestCaseRawResult) (diffResult []claim.TestCaseRawResult, notFoundtestIn1, notFoundtestIn2 []string) {
 	var testcaseR1, testcaseR2 []string
 	for _, result1 := range testcaseResult1 {
 		testcaseR1 = append(testcaseR1, result1.Name)
@@ -195,7 +201,7 @@ func removeDuplicateValues(intSlice []string) []string {
 }
 
 // compare between 2 cni objects and print the difference
-func compare2cni(cni1, cni2 map[string][]Cni) {
+func compare2cni(cni1, cni2 map[string][]claim.Cni) {
 	for node, val := range cni1 {
 		for node2, val2 := range cni2 {
 			if node != node2 {
@@ -220,7 +226,7 @@ func compare2cni(cni1, cni2 map[string][]Cni) {
 // 1. name of cni's that have same name but the plugin value are different - diffPlugins
 // 2. name of cni's that found on claim2 but not in claim1 - notFoundNamesIn1
 // 3. name of cni's that found on claim1 but not in claim2 - notFoundNamesIn3
-func compare2cniHelper(cniList1, cniList2 []Cni, node string) (diffPlugins []Cni, notFoundNamesIn1, notFoundNamesIn2 []string) {
+func compare2cniHelper(cniList1, cniList2 []claim.Cni, node string) (diffPlugins []claim.Cni, notFoundNamesIn1, notFoundNamesIn2 []string) {
 	var cniList1Name, cniList2Name []string
 	if len(cniList1) == 0 {
 		log.Infof("in node %s CNIs present in claim2 and on claim1 that node do not have cni values: %v", node, cniList2)

cmd/tnf/addclaim/compareclaim_test.go → cmd/tnf/claim/compare/compare_test.go

@@ -1,39 +1,41 @@
-package claim
+package compare
 
 import (
 	"reflect"
 	"testing"
+
+	"github.com/test-network-function/cnf-certification-test/cmd/tnf/pkg/claim"
 )
 
 func Test_compare2TestCaseResults(t *testing.T) {
 	type args struct {
-		testcaseResult1 []testCase
-		testcaseResult2 []testCase
+		testcaseResult1 []claim.TestCaseRawResult
+		testcaseResult2 []claim.TestCaseRawResult
 	}
 	tests := []struct {
 		name              string
 		args              args
-		wantDiffresult    []testCase
+		wantDiffresult    []claim.TestCaseRawResult
 		wantNotFoundtest  []string
 		wantNotFoundtest2 []string
 	}{
 		{
 			name: "test1",
 			args: args{
-				testcaseResult1: []testCase{
+				testcaseResult1: []claim.TestCaseRawResult{
 					{
 						Name:   "[It] observability observability-container-logging [common, observability, observability-container-logging]",
 						Status: "skipped",
 					},
 				},
-				testcaseResult2: []testCase{
+				testcaseResult2: []claim.TestCaseRawResult{
 					{
 						Name:   "[It] observability observability-container-logging [common, observability, observability-container-logging]",
 						Status: "failed",
 					},
 				},
 			},
-			wantDiffresult: []testCase{
+			wantDiffresult: []claim.TestCaseRawResult{
 				{
 					Name:   "[It] observability observability-container-logging [common, observability, observability-container-logging]",
 					Status: "skipped",
@@ -45,7 +47,7 @@ func Test_compare2TestCaseResults(t *testing.T) {
 		{
 			name: "test2",
 			args: args{
-				testcaseResult1: []testCase{
+				testcaseResult1: []claim.TestCaseRawResult{
 					{
 						Name:   "[It] observability observability-crd-status [common, observability, observability-crd-status]",
 						Status: "skipped",
@@ -55,14 +57,14 @@ func Test_compare2TestCaseResults(t *testing.T) {
 						Status: "skipped",
 					},
 				},
-				testcaseResult2: []testCase{
+				testcaseResult2: []claim.TestCaseRawResult{
 					{
 						Name:   "[It] observability observability-container-logging [common, observability, observability-container-logging]",
 						Status: "failed",
 					},
 				},
 			},
-			wantDiffresult: []testCase{
+			wantDiffresult: []claim.TestCaseRawResult{
 				{
 					Name:   "[It] observability observability-container-logging [common, observability, observability-container-logging]",
 					Status: "skipped",
@@ -90,21 +92,21 @@ func Test_compare2TestCaseResults(t *testing.T) {
 
 func Test_compare2cnis(t *testing.T) {
 	type args struct {
-		cniList1 []Cni
-		cniList2 []Cni
+		cniList1 []claim.Cni
+		cniList2 []claim.Cni
 		nodeName string
 	}
 	tests := []struct {
 		name               string
 		args               args
-		wantDiffplugins    []Cni
+		wantDiffplugins    []claim.Cni
 		wantNotFoundNames  []string
 		wantNotFoundNames2 []string
 	}{
 		{
 			name: "test1",
 			args: args{
-				cniList1: []Cni{
+				cniList1: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
@@ -114,7 +116,7 @@ func Test_compare2cnis(t *testing.T) {
 						Plugins: nil,
 					},
 				},
-				cniList2: []Cni{
+				cniList2: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
@@ -129,13 +131,13 @@ func Test_compare2cnis(t *testing.T) {
 		{
 			name: "test2",
 			args: args{
-				cniList1: []Cni{
+				cniList1: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
 					},
 				},
-				cniList2: []Cni{
+				cniList2: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
@@ -155,7 +157,7 @@ func Test_compare2cnis(t *testing.T) {
 			name: "test3",
 			args: args{
 				cniList1: nil,
-				cniList2: []Cni{
+				cniList2: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
@@ -185,8 +187,8 @@ func Test_compare2cnis(t *testing.T) {
 		{
 			name: "test5",
 			args: args{
-				cniList1: []Cni{},
-				cniList2: []Cni{},
+				cniList1: []claim.Cni{},
+				cniList2: []claim.Cni{},
 				nodeName: "master1",
 			},
 			wantDiffplugins:    nil,
@@ -196,13 +198,13 @@ func Test_compare2cnis(t *testing.T) {
 		{
 			name: "test6",
 			args: args{
-				cniList1: []Cni{
+				cniList1: []claim.Cni{
 					{
 						Name:    "podman",
 						Plugins: nil,
 					},
 				},
-				cniList2: []Cni{},
+				cniList2: []claim.Cni{},
 				nodeName: "master1",
 			},
 			wantDiffplugins:    nil,