diff --git a/cmd/syft/internal/clio_setup_config.go b/cmd/syft/internal/clio_setup_config.go index 197c640431a..446b7335ce3 100644 --- a/cmd/syft/internal/clio_setup_config.go +++ b/cmd/syft/internal/clio_setup_config.go @@ -6,7 +6,7 @@ import ( "github.com/anchore/clio" "github.com/anchore/stereoscope" - ui2 "github.com/anchore/syft/cmd/syft/cli/ui" + handler "github.com/anchore/syft/cmd/syft/cli/ui" "github.com/anchore/syft/cmd/syft/internal/ui" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" @@ -28,7 +28,7 @@ func AppClioSetupConfig(id clio.Identification, out io.Writer) *clio.SetupConfig return clio.NewUICollection( ui.New(out, cfg.Log.Quiet, - ui2.New(ui2.DefaultHandlerConfig()), + handler.New(handler.DefaultHandlerConfig()), ), noUI, ), nil diff --git a/cmd/syft/internal/commands/cataloger_list.go b/cmd/syft/internal/commands/cataloger_list.go index 2cdf6e8b82f..65c0c73a996 100644 --- a/cmd/syft/internal/commands/cataloger_list.go +++ b/cmd/syft/internal/commands/cataloger_list.go @@ -3,6 +3,7 @@ package commands import ( "encoding/json" "fmt" + "os" "sort" "strings" @@ -14,7 +15,15 @@ import ( "github.com/anchore/clio" "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/task" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/cataloging" +) + +var ( + activelyAddedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green + deselectedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey + activelyRemovedStyle = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red + defaultStyle = lipgloss.NewStyle().Underline(true) + deselectedDefaultStyle = lipgloss.NewStyle().Inherit(deselectedStyle).Underline(true) ) type catalogerListOptions struct { @@ -44,8 +53,9 @@ func CatalogerList(app clio.Application) *cobra.Command { opts := defaultCatalogerListOptions() return app.SetupCommand(&cobra.Command{ - Use: "list [OPTIONS]", - Short: "List available catalogers", + Use: "list [OPTIONS]", + Short: "List available catalogers", + PreRunE: disableUI(app, os.Stdout), RunE: func(_ *cobra.Command, _ []string) error { return runCatalogerList(opts) }, @@ -53,13 +63,19 @@ func CatalogerList(app clio.Application) *cobra.Command { } func runCatalogerList(opts *catalogerListOptions) error { - factories := task.DefaultPackageTaskFactories() - allTasks, err := factories.Tasks(task.DefaultCatalogingFactoryConfig()) + pkgTaskFactories := task.DefaultPackageTaskFactories() + fileTaskFactories := task.DefaultFileTaskFactories() + allPkgTasks, err := pkgTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig()) + if err != nil { + return fmt.Errorf("unable to create pkg cataloger tasks: %w", err) + } + + allFileTasks, err := fileTaskFactories.Tasks(task.DefaultCatalogingFactoryConfig()) if err != nil { - return fmt.Errorf("unable to create cataloger tasks: %w", err) + return fmt.Errorf("unable to create file cataloger tasks: %w", err) } - report, err := catalogerListReport(opts, allTasks) + report, err := catalogerListReport(opts, [][]task.Task{allPkgTasks, allFileTasks}) if err != nil { return fmt.Errorf("unable to generate cataloger list report: %w", err) } @@ -69,9 +85,10 @@ func runCatalogerList(opts *catalogerListOptions) error { return nil } -func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (string, error) { - selectedTasks, selectionEvidence, err := task.Select(allTasks, - pkgcataloging.NewSelectionRequest(). +func catalogerListReport(opts *catalogerListOptions, allTaskGroups [][]task.Task) (string, error) { + selectedTaskGroups, selectionEvidence, err := task.SelectInGroups( + allTaskGroups, + cataloging.NewSelectionRequest(). WithDefaults(opts.DefaultCatalogers...). WithExpression(opts.SelectCatalogers...), ) @@ -82,12 +99,12 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri switch opts.Output { case "json": - report, err = renderCatalogerListJSON(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) + report, err = renderCatalogerListJSON(flattenTaskGroups(selectedTaskGroups), selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) case "table", "": if opts.ShowHidden { - report = renderCatalogerListTable(allTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) + report = renderCatalogerListTables(allTaskGroups, selectionEvidence) } else { - report = renderCatalogerListTable(selectedTasks, selectionEvidence, opts.DefaultCatalogers, opts.SelectCatalogers) + report = renderCatalogerListTables(selectedTaskGroups, selectionEvidence) } } @@ -98,6 +115,14 @@ func catalogerListReport(opts *catalogerListOptions, allTasks []task.Task) (stri return report, nil } +func flattenTaskGroups(taskGroups [][]task.Task) []task.Task { + var allTasks []task.Task + for _, tasks := range taskGroups { + allTasks = append(allTasks, tasks...) + } + return allTasks +} + func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) (string, error) { type node struct { Name string `json:"name"` @@ -109,7 +134,12 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul nodesByName := make(map[string]node) for name := range tagsByName { - tagsSelected := selection.TokensByTask[name].SelectedOn.List() + tokensByTask, ok := selection.TokensByTask[name] + + var tagsSelected []string + if ok { + tagsSelected = tokensByTask.SelectedOn.List() + } if len(tagsSelected) == 1 && tagsSelected[0] == "all" { tagsSelected = tagsByName[name] @@ -153,10 +183,56 @@ func renderCatalogerListJSON(tasks []task.Task, selection task.Selection, defaul return string(by), err } -func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defaultSelections, selections []string) string { +func renderCatalogerListTables(taskGroups [][]task.Task, selection task.Selection) string { + pkgCatalogerTable := renderCatalogerListTable(taskGroups[0], selection, "Package Cataloger") + fileCatalogerTable := renderCatalogerListTable(taskGroups[1], selection, "File Cataloger") + + report := fileCatalogerTable + "\n" + pkgCatalogerTable + "\n" + + hasAdditions := len(selection.Request.AddNames) > 0 + hasDefaults := len(selection.Request.DefaultNamesOrTags) > 0 + hasRemovals := len(selection.Request.RemoveNamesOrTags) > 0 + hasSubSelections := len(selection.Request.SubSelectTags) > 0 + expressions := len(selection.Request.SubSelectTags) + len(selection.Request.AddNames) + len(selection.Request.RemoveNamesOrTags) + + var header string + + header += fmt.Sprintf("Default selections: %d\n", len(selection.Request.DefaultNamesOrTags)) + if hasDefaults { + for _, expr := range selection.Request.DefaultNamesOrTags { + header += fmt.Sprintf(" • '%s'\n", expr) + } + } + + header += fmt.Sprintf("Selection expressions: %d\n", expressions) + + if hasSubSelections { + for _, n := range selection.Request.SubSelectTags { + header += fmt.Sprintf(" • '%s' (intersect)\n", n) + } + } + if hasRemovals { + for _, n := range selection.Request.RemoveNamesOrTags { + header += fmt.Sprintf(" • '-%s' (remove)\n", n) + } + } + if hasAdditions { + for _, n := range selection.Request.AddNames { + header += fmt.Sprintf(" • '+%s' (add)\n", n) + } + } + + return header + report +} + +func renderCatalogerListTable(tasks []task.Task, selection task.Selection, kindTitle string) string { + if len(tasks) == 0 { + return activelyRemovedStyle.Render(fmt.Sprintf("No %ss selected", strings.ToLower(kindTitle))) + } + t := table.NewWriter() t.SetStyle(table.StyleLight) - t.AppendHeader(table.Row{"Cataloger", "Tags"}) + t.AppendHeader(table.Row{kindTitle, "Tags"}) names, tagsByName := extractTaskInfo(tasks) @@ -172,27 +248,12 @@ func renderCatalogerListTable(tasks []task.Task, selection task.Selection, defau report := t.Render() - if len(selections) > 0 { - header := "Selected by expressions:\n" - for _, expr := range selections { - header += fmt.Sprintf(" - %q\n", expr) - } - report = header + report - } - - if len(defaultSelections) > 0 { - header := "Default selections:\n" - for _, expr := range defaultSelections { - header += fmt.Sprintf(" - %q\n", expr) - } - report = header + report - } - return report } func formatRow(name string, tags []string, selection task.Selection) table.Row { isIncluded := selection.Result.Has(name) + defaults := strset.New(selection.Request.DefaultNamesOrTags...) var selections *task.TokenSelection if s, exists := selection.TokensByTask[name]; exists { selections = &s @@ -200,35 +261,32 @@ func formatRow(name string, tags []string, selection task.Selection) table.Row { var formattedTags []string for _, tag := range tags { - formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded)) + formattedTags = append(formattedTags, formatToken(tag, selections, isIncluded, defaults)) } var tagStr string if isIncluded { tagStr = strings.Join(formattedTags, ", ") } else { - tagStr = strings.Join(formattedTags, grey.Render(", ")) + tagStr = strings.Join(formattedTags, deselectedStyle.Render(", ")) } // TODO: selection should keep warnings (non-selections) in struct return table.Row{ - formatToken(name, selections, isIncluded), + formatToken(name, selections, isIncluded, defaults), tagStr, } } -var ( - green = lipgloss.NewStyle().Foreground(lipgloss.Color("10")) // hi green - grey = lipgloss.NewStyle().Foreground(lipgloss.Color("8")) // dark grey - red = lipgloss.NewStyle().Foreground(lipgloss.Color("9")) // high red -) - -func formatToken(token string, selection *task.TokenSelection, included bool) string { +func formatToken(token string, selection *task.TokenSelection, included bool, defaults *strset.Set) string { if included && selection != nil { // format all tokens in selection in green if selection.SelectedOn.Has(token) { - return green.Render(token) + if defaults.Has(token) { + return defaultStyle.Render(token) + } + return activelyAddedStyle.Render(token) } return token @@ -236,10 +294,12 @@ func formatToken(token string, selection *task.TokenSelection, included bool) st // format all tokens in selection in red, all others in grey if selection != nil && selection.DeselectedOn.Has(token) { - return red.Render(token) + return activelyRemovedStyle.Render(token) } - - return grey.Render(token) + if defaults.Has(token) { + return deselectedDefaultStyle.Render(token) + } + return deselectedStyle.Render(token) } func extractTaskInfo(tasks []task.Task) ([]string, map[string][]string) { diff --git a/cmd/syft/internal/commands/cataloger_list_test.go b/cmd/syft/internal/commands/cataloger_list_test.go index 24d8ab1f20c..5a1e5f95695 100644 --- a/cmd/syft/internal/commands/cataloger_list_test.go +++ b/cmd/syft/internal/commands/cataloger_list_test.go @@ -40,23 +40,35 @@ func (d dummyTask) Execute(_ context.Context, _ file.Resolver, _ sbomsync.Builde panic("implement me") } -func testTasks() []task.Task { - return []task.Task{ - dummyTask{ - name: "task1", - selectors: []string{"image", "a", "b", "1"}, - }, - dummyTask{ - name: "task2", - selectors: []string{"image", "b", "c", "2"}, - }, - dummyTask{ - name: "task3", - selectors: []string{"directory", "c", "d", "3"}, +func testTasks() [][]task.Task { + return [][]task.Task{ + { + dummyTask{ + name: "task1", + selectors: []string{"image", "a", "b", "1"}, + }, + dummyTask{ + name: "task2", + selectors: []string{"image", "b", "c", "2"}, + }, + dummyTask{ + name: "task3", + selectors: []string{"directory", "c", "d", "3"}, + }, + dummyTask{ + name: "task4", + selectors: []string{"directory", "d", "e", "4"}, + }, }, - dummyTask{ - name: "task4", - selectors: []string{"directory", "d", "e", "4"}, + { + dummyTask{ + name: "file-task1", + selectors: []string{"file", "ft", "ft-1-b"}, + }, + dummyTask{ + name: "file-task2", + selectors: []string{"file", "ft", "ft-2-b"}, + }, }, } } @@ -76,16 +88,23 @@ func Test_catalogerListReport(t *testing.T) { return c }(), want: ` -Default selections: - - "all" -┌───────────┬────────────────────┐ -│ CATALOGER │ TAGS │ -├───────────┼────────────────────┤ -│ task1 │ 1, a, b, image │ -│ task2 │ 2, b, c, image │ -│ task3 │ 3, c, d, directory │ -│ task4 │ 4, d, directory, e │ -└───────────┴────────────────────┘ +Default selections: 1 + • 'all' +Selection expressions: 0 +┌────────────────┬──────────────────┐ +│ FILE CATALOGER │ TAGS │ +├────────────────┼──────────────────┤ +│ file-task1 │ file, ft, ft-1-b │ +│ file-task2 │ file, ft, ft-2-b │ +└────────────────┴──────────────────┘ +┌───────────────────┬────────────────────┐ +│ PACKAGE CATALOGER │ TAGS │ +├───────────────────┼────────────────────┤ +│ task1 │ 1, a, b, image │ +│ task2 │ 2, b, c, image │ +│ task3 │ 3, c, d, directory │ +│ task4 │ 4, d, directory, e │ +└───────────────────┴────────────────────┘ `, }, { @@ -96,7 +115,7 @@ Default selections: return c }(), want: ` -{"default":["all"],"selection":[],"catalogers":[{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]} +{"default":["all"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file","ft","ft-1-b"]},{"name":"file-task2","tags":["file","ft","ft-2-b"]},{"name":"task1","tags":["1","a","b","image"]},{"name":"task2","tags":["2","b","c","image"]},{"name":"task3","tags":["3","c","d","directory"]},{"name":"task4","tags":["4","d","directory","e"]}]} `, }, { @@ -105,19 +124,27 @@ Default selections: c := defaultCatalogerListOptions() c.Output = "table" c.DefaultCatalogers = []string{ - "image", + "image", // note: for backwards compatibility file will automatically be added } return c }(), want: ` -Default selections: - - "image" -┌───────────┬────────────────┐ -│ CATALOGER │ TAGS │ -├───────────┼────────────────┤ -│ task1 │ 1, a, b, image │ -│ task2 │ 2, b, c, image │ -└───────────┴────────────────┘ +Default selections: 2 + • 'image' + • 'file' +Selection expressions: 0 +┌────────────────┬──────────────────┐ +│ FILE CATALOGER │ TAGS │ +├────────────────┼──────────────────┤ +│ file-task1 │ file, ft, ft-1-b │ +│ file-task2 │ file, ft, ft-2-b │ +└────────────────┴──────────────────┘ +┌───────────────────┬────────────────┐ +│ PACKAGE CATALOGER │ TAGS │ +├───────────────────┼────────────────┤ +│ task1 │ 1, a, b, image │ +│ task2 │ 2, b, c, image │ +└───────────────────┴────────────────┘ `, }, { @@ -131,7 +158,7 @@ Default selections: return c }(), want: ` -{"default":["image"],"selection":[],"catalogers":[{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]} +{"default":["image"],"selection":[],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["image"]},{"name":"task2","tags":["image"]}]} `, }, { @@ -147,23 +174,33 @@ Default selections: "+task3", "-c", "b", + "-file", + "+file-task1", } return c }(), want: ` -Default selections: - - "image" -Selected by expressions: - - "-directory" - - "+task3" - - "-c" - - "b" -┌───────────┬────────────────────┐ -│ CATALOGER │ TAGS │ -├───────────┼────────────────────┤ -│ task1 │ 1, a, b, image │ -│ task3 │ 3, c, d, directory │ -└───────────┴────────────────────┘ +Default selections: 2 + • 'image' + • 'file' +Selection expressions: 6 + • 'b' (intersect) + • '-directory' (remove) + • '-c' (remove) + • '-file' (remove) + • '+task3' (add) + • '+file-task1' (add) +┌────────────────┬──────────────────┐ +│ FILE CATALOGER │ TAGS │ +├────────────────┼──────────────────┤ +│ file-task1 │ file, ft, ft-1-b │ +└────────────────┴──────────────────┘ +┌───────────────────┬────────────────────┐ +│ PACKAGE CATALOGER │ TAGS │ +├───────────────────┼────────────────────┤ +│ task1 │ 1, a, b, image │ +│ task3 │ 3, c, d, directory │ +└───────────────────┴────────────────────┘ `, }, { @@ -183,7 +220,7 @@ Selected by expressions: return c }(), want: ` -{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]} +{"default":["image"],"selection":["-directory","+task3","-c","b"],"catalogers":[{"name":"file-task1","tags":["file"]},{"name":"file-task2","tags":["file"]},{"name":"task1","tags":["b","image"]},{"name":"task3","tags":["task3"]}]} `, }, } diff --git a/cmd/syft/internal/commands/utils.go b/cmd/syft/internal/commands/utils.go new file mode 100644 index 00000000000..bc0e6b1fc00 --- /dev/null +++ b/cmd/syft/internal/commands/utils.go @@ -0,0 +1,23 @@ +package commands + +import ( + "io" + + "github.com/spf13/cobra" + + "github.com/anchore/clio" + "github.com/anchore/syft/cmd/syft/internal/ui" +) + +func disableUI(app clio.Application, out io.Writer) func(*cobra.Command, []string) error { + return func(_ *cobra.Command, _ []string) error { + type Stater interface { + State() *clio.State + } + + state := app.(Stater).State() + state.UI = clio.NewUICollection(ui.None(out, state.Config.Log.Quiet)) + + return nil + } +} diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 22cb286a445..217b0fc6ee0 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -91,7 +91,7 @@ func (cfg Catalog) ToSBOMConfig(id clio.Identification) *syft.CreateSBOMConfig { WithPackagesConfig(cfg.ToPackagesConfig()). WithFilesConfig(cfg.ToFilesConfig()). WithCatalogerSelection( - pkgcataloging.NewSelectionRequest(). + cataloging.NewSelectionRequest(). WithDefaults(cfg.DefaultCatalogers...). WithExpression(cfg.SelectCatalogers...), ) diff --git a/cmd/syft/internal/options/source.go b/cmd/syft/internal/options/source.go index 53078305c2e..97e81b9459f 100644 --- a/cmd/syft/internal/options/source.go +++ b/cmd/syft/internal/options/source.go @@ -5,11 +5,11 @@ import ( "sort" "strings" - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/dustin/go-humanize" "github.com/scylladb/go-set/strset" "github.com/anchore/clio" + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/syft/syft/source/sourceproviders" ) diff --git a/cmd/syft/internal/test/integration/utils_test.go b/cmd/syft/internal/test/integration/utils_test.go index 4e1ffb2096f..978b8a7cc68 100644 --- a/cmd/syft/internal/test/integration/utils_test.go +++ b/cmd/syft/internal/test/integration/utils_test.go @@ -10,6 +10,7 @@ import ( "github.com/anchore/stereoscope/pkg/imagetest" "github.com/anchore/syft/cmd/syft/internal/options" "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" @@ -20,7 +21,7 @@ func catalogFixtureImage(t *testing.T, fixtureImageName string, scope source.Sco Name: "syft-tester", Version: "v0.99.0", }).WithCatalogerSelection( - pkgcataloging.NewSelectionRequest(). + cataloging.NewSelectionRequest(). WithExpression(catalogerSelection...), ) cfg.Search.Scope = scope @@ -55,7 +56,7 @@ func catalogDirectory(t *testing.T, dir string, catalogerSelection ...string) (s Name: "syft-tester", Version: "v0.99.0", }).WithCatalogerSelection( - pkgcataloging.NewSelectionRequest(). + cataloging.NewSelectionRequest(). WithExpression(catalogerSelection...), ) diff --git a/examples/create_custom_sbom/main.go b/examples/create_custom_sbom/main.go index aa51ffc0f9e..f678549d3f5 100644 --- a/examples/create_custom_sbom/main.go +++ b/examples/create_custom_sbom/main.go @@ -74,7 +74,7 @@ func getSBOM(src source.Source) sbom.SBOM { // only use OS related catalogers that would have been used with the kind of // source type (container image or directory), but also add a specific python cataloger WithCatalogerSelection( - pkgcataloging.NewSelectionRequest(). + cataloging.NewSelectionRequest(). WithSubSelections("os"). WithAdditions("python-package-cataloger"), ). diff --git a/examples/select_catalogers/main.go b/examples/select_catalogers/main.go index 79cdd440049..6bb8fa2341c 100644 --- a/examples/select_catalogers/main.go +++ b/examples/select_catalogers/main.go @@ -6,6 +6,7 @@ import ( "os" "github.com/anchore/syft/syft" + "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/pkgcataloging" "github.com/anchore/syft/syft/sbom" "github.com/anchore/syft/syft/source" @@ -55,7 +56,7 @@ func getSBOM(src source.Source, defaultTags ...string) sbom.SBOM { WithCatalogerSelection( // here you can sub-select, add, remove catalogers from the default selection... // or replace the default selection entirely! - pkgcataloging.NewSelectionRequest(). + cataloging.NewSelectionRequest(). WithDefaults(defaultTags...), ) diff --git a/internal/task/cataloging_config.go b/internal/task/cataloging_config.go new file mode 100644 index 00000000000..bef9d673c64 --- /dev/null +++ b/internal/task/cataloging_config.go @@ -0,0 +1,27 @@ +package task + +import ( + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/cataloging/filecataloging" + "github.com/anchore/syft/syft/cataloging/pkgcataloging" +) + +type CatalogingFactoryConfig struct { + ComplianceConfig cataloging.ComplianceConfig + SearchConfig cataloging.SearchConfig + RelationshipsConfig cataloging.RelationshipsConfig + DataGenerationConfig cataloging.DataGenerationConfig + PackagesConfig pkgcataloging.Config + FilesConfig filecataloging.Config +} + +func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig { + return CatalogingFactoryConfig{ + ComplianceConfig: cataloging.DefaultComplianceConfig(), + SearchConfig: cataloging.DefaultSearchConfig(), + RelationshipsConfig: cataloging.DefaultRelationshipsConfig(), + DataGenerationConfig: cataloging.DefaultDataGenerationConfig(), + PackagesConfig: pkgcataloging.DefaultConfig(), + FilesConfig: filecataloging.DefaultConfig(), + } +} diff --git a/internal/task/expression.go b/internal/task/expression.go index e99551c6f3c..5be6a4714a0 100644 --- a/internal/task/expression.go +++ b/internal/task/expression.go @@ -8,7 +8,7 @@ import ( "github.com/hashicorp/go-multierror" "github.com/scylladb/go-set/strset" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/cataloging" ) var expressionNodePattern = regexp.MustCompile(`^([a-zA-Z0-9][a-zA-Z0-9-+]*)+$`) @@ -142,7 +142,7 @@ func (ec expressionContext) newExpression(exp string, operation Operation, token } } -func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest pkgcataloging.SelectionRequest) Expressions { +func newExpressionsFromSelectionRequest(nc *expressionContext, selectionRequest cataloging.SelectionRequest) Expressions { var all Expressions for _, exp := range selectionRequest.DefaultNamesOrTags { diff --git a/internal/task/expression_test.go b/internal/task/expression_test.go index e1a97ab2213..d9835c99726 100644 --- a/internal/task/expression_test.go +++ b/internal/task/expression_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/cataloging" ) func Test_newExpressionsFromSelectionRequest(t *testing.T) { @@ -135,7 +135,7 @@ func Test_newExpressionsFromSelectionRequest(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) + req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) result := newExpressionsFromSelectionRequest(nc, req) if tt.expectedErrors != nil { diff --git a/internal/task/factory.go b/internal/task/factory.go new file mode 100644 index 00000000000..c51c1975b78 --- /dev/null +++ b/internal/task/factory.go @@ -0,0 +1,40 @@ +package task + +import ( + "fmt" + "sort" + "strings" + + "github.com/scylladb/go-set/strset" +) + +type factory func(cfg CatalogingFactoryConfig) Task + +type Factories []factory + +func (f Factories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) { + var allTasks []Task + taskNames := strset.New() + duplicateTaskNames := strset.New() + var err error + for _, fact := range f { + tsk := fact(cfg) + if tsk == nil { + continue + } + tskName := tsk.Name() + if taskNames.Has(tskName) { + duplicateTaskNames.Add(tskName) + } + + allTasks = append(allTasks, tsk) + taskNames.Add(tskName) + } + if duplicateTaskNames.Size() > 0 { + names := duplicateTaskNames.List() + sort.Strings(names) + err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", ")) + } + + return allTasks, err +} diff --git a/internal/task/file_tasks.go b/internal/task/file_tasks.go index 5b6f7bbd4dd..2ee08f72be8 100644 --- a/internal/task/file_tasks.go +++ b/internal/task/file_tasks.go @@ -6,6 +6,7 @@ import ( "github.com/anchore/syft/internal/sbomsync" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cataloging/filecataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/file/cataloger/executable" "github.com/anchore/syft/syft/file/cataloger/filecontent" @@ -15,14 +16,27 @@ import ( "github.com/anchore/syft/syft/sbom" ) -func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash) Task { - if selection == file.NoFilesSelection || len(hashers) == 0 { - return nil +func DefaultFileTaskFactories() Factories { + return Factories{ + newFileDigestCatalogerTaskFactory("digest"), + newFileMetadataCatalogerTaskFactory("file-metadata"), + newFileContentCatalogerTaskFactory("content"), + newExecutableCatalogerTaskFactory("binary-metadata"), } +} - digestsCataloger := filedigest.NewCataloger(hashers) +func newFileDigestCatalogerTaskFactory(tags ...string) factory { + return func(cfg CatalogingFactoryConfig) Task { + return newFileDigestCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Hashers, tags...) + } +} +func newFileDigestCatalogerTask(selection file.Selection, hashers []crypto.Hash, tags ...string) Task { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + if selection == file.NoFilesSelection || len(hashers) == 0 { + return nil + } + accessor := builder.(sbomsync.Accessor) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) @@ -30,7 +44,7 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash return nil } - result, err := digestsCataloger.Catalog(ctx, resolver, coordinates...) + result, err := filedigest.NewCataloger(hashers).Catalog(ctx, resolver, coordinates...) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { sbom.Artifacts.FileDigests = result @@ -39,17 +53,21 @@ func NewFileDigestCatalogerTask(selection file.Selection, hashers ...crypto.Hash return err } - return NewTask("file-digest-cataloger", fn) + return NewTask("file-digest-cataloger", fn, commonFileTags(tags)...) } -func NewFileMetadataCatalogerTask(selection file.Selection) Task { - if selection == file.NoFilesSelection { - return nil +func newFileMetadataCatalogerTaskFactory(tags ...string) factory { + return func(cfg CatalogingFactoryConfig) Task { + return newFileMetadataCatalogerTask(cfg.FilesConfig.Selection, tags...) } +} - metadataCataloger := filemetadata.NewCataloger() - +func newFileMetadataCatalogerTask(selection file.Selection, tags ...string) Task { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + if selection == file.NoFilesSelection { + return nil + } + accessor := builder.(sbomsync.Accessor) coordinates, ok := coordinatesForSelection(selection, builder.(sbomsync.Accessor)) @@ -57,7 +75,7 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task { return nil } - result, err := metadataCataloger.Catalog(ctx, resolver, coordinates...) + result, err := filemetadata.NewCataloger().Catalog(ctx, resolver, coordinates...) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { sbom.Artifacts.FileMetadata = result @@ -66,20 +84,24 @@ func NewFileMetadataCatalogerTask(selection file.Selection) Task { return err } - return NewTask("file-metadata-cataloger", fn) + return NewTask("file-metadata-cataloger", fn, commonFileTags(tags)...) } -func NewFileContentCatalogerTask(cfg filecontent.Config) Task { - if len(cfg.Globs) == 0 { - return nil +func newFileContentCatalogerTaskFactory(tags ...string) factory { + return func(cfg CatalogingFactoryConfig) Task { + return newFileContentCatalogerTask(cfg.FilesConfig.Content, tags...) } +} - cat := filecontent.NewCataloger(cfg) - +func newFileContentCatalogerTask(cfg filecontent.Config, tags ...string) Task { fn := func(ctx context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + if len(cfg.Globs) == 0 { + return nil + } + accessor := builder.(sbomsync.Accessor) - result, err := cat.Catalog(ctx, resolver) + result, err := filecontent.NewCataloger(cfg).Catalog(ctx, resolver) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { sbom.Artifacts.FileContents = result @@ -88,20 +110,24 @@ func NewFileContentCatalogerTask(cfg filecontent.Config) Task { return err } - return NewTask("file-content-cataloger", fn) + return NewTask("file-content-cataloger", fn, commonFileTags(tags)...) } -func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config) Task { - if selection == file.NoFilesSelection { - return nil +func newExecutableCatalogerTaskFactory(tags ...string) factory { + return func(cfg CatalogingFactoryConfig) Task { + return newExecutableCatalogerTask(cfg.FilesConfig.Selection, cfg.FilesConfig.Executable, tags...) } +} - cat := executable.NewCataloger(cfg) - +func newExecutableCatalogerTask(selection file.Selection, cfg executable.Config, tags ...string) Task { fn := func(_ context.Context, resolver file.Resolver, builder sbomsync.Builder) error { + if selection == file.NoFilesSelection { + return nil + } + accessor := builder.(sbomsync.Accessor) - result, err := cat.Catalog(resolver) + result, err := executable.NewCataloger(cfg).Catalog(resolver) accessor.WriteToSBOM(func(sbom *sbom.SBOM) { sbom.Artifacts.Executables = result @@ -110,7 +136,7 @@ func NewExecutableCatalogerTask(selection file.Selection, cfg executable.Config) return err } - return NewTask("file-executable-cataloger", fn) + return NewTask("file-executable-cataloger", fn, commonFileTags(tags)...) } // TODO: this should be replaced with a fix that allows passing a coordinate or location iterator to the cataloger @@ -154,3 +180,8 @@ func coordinatesForSelection(selection file.Selection, accessor sbomsync.Accesso return nil, false } + +func commonFileTags(tags []string) []string { + tags = append(tags, filecataloging.FileTag) + return tags +} diff --git a/internal/task/package_task_factory.go b/internal/task/package_task_factory.go index 5a65a29f040..bb178c946e1 100644 --- a/internal/task/package_task_factory.go +++ b/internal/task/package_task_factory.go @@ -3,12 +3,9 @@ package task import ( "context" "fmt" - "sort" "strings" "unicode" - "github.com/scylladb/go-set/strset" - "github.com/anchore/syft/internal/bus" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/relationship" @@ -23,67 +20,18 @@ import ( cpeutils "github.com/anchore/syft/syft/pkg/cataloger/common/cpe" ) -type packageTaskFactory func(cfg CatalogingFactoryConfig) Task - -type PackageTaskFactories []packageTaskFactory - -type CatalogingFactoryConfig struct { - ComplianceConfig cataloging.ComplianceConfig - SearchConfig cataloging.SearchConfig - RelationshipsConfig cataloging.RelationshipsConfig - DataGenerationConfig cataloging.DataGenerationConfig - PackagesConfig pkgcataloging.Config -} - -func DefaultCatalogingFactoryConfig() CatalogingFactoryConfig { - return CatalogingFactoryConfig{ - ComplianceConfig: cataloging.DefaultComplianceConfig(), - SearchConfig: cataloging.DefaultSearchConfig(), - RelationshipsConfig: cataloging.DefaultRelationshipsConfig(), - DataGenerationConfig: cataloging.DefaultDataGenerationConfig(), - PackagesConfig: pkgcataloging.DefaultConfig(), - } -} - -func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) packageTaskFactory { +func newPackageTaskFactory(catalogerFactory func(CatalogingFactoryConfig) pkg.Cataloger, tags ...string) factory { return func(cfg CatalogingFactoryConfig) Task { return NewPackageTask(cfg, catalogerFactory(cfg), tags...) } } -func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) packageTaskFactory { +func newSimplePackageTaskFactory(catalogerFactory func() pkg.Cataloger, tags ...string) factory { return func(cfg CatalogingFactoryConfig) Task { return NewPackageTask(cfg, catalogerFactory(), tags...) } } -func (f PackageTaskFactories) Tasks(cfg CatalogingFactoryConfig) ([]Task, error) { - var allTasks []Task - taskNames := strset.New() - duplicateTaskNames := strset.New() - var err error - for _, factory := range f { - tsk := factory(cfg) - if tsk == nil { - continue - } - tskName := tsk.Name() - if taskNames.Has(tskName) { - duplicateTaskNames.Add(tskName) - } - - allTasks = append(allTasks, tsk) - taskNames.Add(tskName) - } - if duplicateTaskNames.Size() > 0 { - names := duplicateTaskNames.List() - sort.Strings(names) - err = fmt.Errorf("duplicate cataloger task names: %v", strings.Join(names, ", ")) - } - - return allTasks, err -} - // NewPackageTask creates a Task function for a generic pkg.Cataloger, honoring the common configuration options. func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string) Task { fn := func(ctx context.Context, resolver file.Resolver, sbom sbomsync.Builder) error { diff --git a/internal/task/package_tasks.go b/internal/task/package_tasks.go index 431c77590f4..1f7488e9e20 100644 --- a/internal/task/package_tasks.go +++ b/internal/task/package_tasks.go @@ -50,8 +50,8 @@ const ( ) //nolint:funlen -func DefaultPackageTaskFactories() PackageTaskFactories { - return []packageTaskFactory{ +func DefaultPackageTaskFactories() Factories { + return []factory{ // OS package installed catalogers /////////////////////////////////////////////////////////////////////////// newSimplePackageTaskFactory(arch.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "alpm", "archlinux"), newSimplePackageTaskFactory(alpine.NewDBCataloger, pkgcataloging.DirectoryTag, pkgcataloging.InstalledTag, pkgcataloging.ImageTag, pkgcataloging.OSTag, "linux", "apk", "alpine"), diff --git a/internal/task/selection.go b/internal/task/selection.go index bd0cacdfbeb..118a7ab1d5a 100644 --- a/internal/task/selection.go +++ b/internal/task/selection.go @@ -3,17 +3,19 @@ package task import ( "fmt" "sort" + "strings" "github.com/scylladb/go-set/strset" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/cataloging" + "github.com/anchore/syft/syft/cataloging/filecataloging" ) // Selection represents the users request for a subset of tasks to run and the resulting set of task names that were // selected. Additionally, all tokens that were matched on to reach the returned conclusion are also provided. type Selection struct { - Request pkgcataloging.SelectionRequest + Request cataloging.SelectionRequest Result *strset.Set TokensByTask map[string]TokenSelection } @@ -52,7 +54,13 @@ func newSelection() Selection { // Select parses the given expressions as two sets: expressions that represent a "set" operation, and expressions that // represent all other operations. The parsed expressions are then evaluated against the given tasks to return // a subset (or the same) set of tasks. -func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([]Task, Selection, error) { +func Select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) { + ensureDefaultSelectionHasFiles(&selectionRequest, allTasks) + + return _select(allTasks, selectionRequest) +} + +func _select(allTasks []Task, selectionRequest cataloging.SelectionRequest) ([]Task, Selection, error) { nodes := newExpressionsFromSelectionRequest(newExpressionContext(allTasks), selectionRequest) finalTasks, selection := selectByExpressions(allTasks, nodes) @@ -62,6 +70,142 @@ func Select(allTasks []Task, selectionRequest pkgcataloging.SelectionRequest) ([ return finalTasks, selection, nodes.Validate() } +// ensureDefaultSelectionHasFiles ensures that the default selection request has the "file" tag, as this is a required +// for backwards compatibility (when catalogers we only for packages and not for separate groups of tasks). +func ensureDefaultSelectionHasFiles(selectionRequest *cataloging.SelectionRequest, allTasks ...[]Task) { + for _, ts := range allTasks { + _, leftOver := tagsOrNamesThatTaskGroupRespondsTo(ts, strset.New(filecataloging.FileTag)) + if leftOver.Has(filecataloging.FileTag) { + // the given set of tasks do not respond to file, so don't include it in the default selection + continue + } + + defaultNamesOrTags := strset.New(selectionRequest.DefaultNamesOrTags...) + removals := strset.New(selectionRequest.RemoveNamesOrTags...) + missingFileIshTag := !defaultNamesOrTags.Has(filecataloging.FileTag) && !defaultNamesOrTags.Has("all") && !defaultNamesOrTags.Has("default") + if missingFileIshTag && !removals.Has(filecataloging.FileTag) { + log.Warnf("adding '%s' tag to the default cataloger selection, to override add '-%s' to the cataloger selection request", filecataloging.FileTag, filecataloging.FileTag) + selectionRequest.DefaultNamesOrTags = append(selectionRequest.DefaultNamesOrTags, filecataloging.FileTag) + } + } +} + +// SelectInGroups is a convenience function that allows for selecting tasks from multiple groups of tasks. The original +// request is splut into sub-requests, where only tokens that are relevant to the given group of tasks are considered. +// If tokens are passed that are not relevant to any group of tasks, an error is returned. +func SelectInGroups(taskGroups [][]Task, selectionRequest cataloging.SelectionRequest) ([][]Task, Selection, error) { + ensureDefaultSelectionHasFiles(&selectionRequest, taskGroups...) + + reqs, errs := splitCatalogerSelectionRequest(selectionRequest, taskGroups) + if errs != nil { + return nil, Selection{ + Request: selectionRequest, + }, errs + } + + var finalTasks [][]Task + var selections []Selection + for idx, req := range reqs { + tskGroup := taskGroups[idx] + subFinalTasks, subSelection, err := _select(tskGroup, req) + if err != nil { + return nil, Selection{ + Request: selectionRequest, + }, err + } + finalTasks = append(finalTasks, subFinalTasks) + selections = append(selections, subSelection) + } + + return finalTasks, mergeSelections(selections, selectionRequest), nil +} + +func mergeSelections(selections []Selection, ogRequest cataloging.SelectionRequest) Selection { + finalSelection := newSelection() + for _, s := range selections { + finalSelection.Result.Add(s.Result.List()...) + for name, tokenSelection := range s.TokensByTask { + if existing, exists := finalSelection.TokensByTask[name]; exists { + existing.merge(tokenSelection) + finalSelection.TokensByTask[name] = existing + } else { + finalSelection.TokensByTask[name] = tokenSelection + } + } + } + finalSelection.Request = ogRequest + return finalSelection +} + +func splitCatalogerSelectionRequest(req cataloging.SelectionRequest, selectablePkgTaskGroups [][]Task) ([]cataloging.SelectionRequest, error) { + requestTagsOrNames := allRequestReferences(req) + leftoverTags := strset.New() + usedTagsAndNames := strset.New() + var usedTagGroups []*strset.Set + for _, taskGroup := range selectablePkgTaskGroups { + selectedTagOrNames, remainingTagsOrNames := tagsOrNamesThatTaskGroupRespondsTo(taskGroup, requestTagsOrNames) + leftoverTags = strset.Union(leftoverTags, remainingTagsOrNames) + usedTagGroups = append(usedTagGroups, selectedTagOrNames) + usedTagsAndNames.Add(selectedTagOrNames.List()...) + } + + leftoverTags = strset.Difference(leftoverTags, usedTagsAndNames) + leftoverTags.Remove("all") + + if leftoverTags.Size() > 0 { + l := leftoverTags.List() + sort.Strings(l) + return nil, fmt.Errorf("no cataloger tasks respond to the following selections: %v", strings.Join(l, ", ")) + } + + var newSelections []cataloging.SelectionRequest + for _, tags := range usedTagGroups { + newSelections = append(newSelections, newSelectionWithTags(req, tags)) + } + + return newSelections, nil +} + +func newSelectionWithTags(req cataloging.SelectionRequest, tags *strset.Set) cataloging.SelectionRequest { + return cataloging.SelectionRequest{ + DefaultNamesOrTags: filterTags(req.DefaultNamesOrTags, tags), + SubSelectTags: filterTags(req.SubSelectTags, tags), + AddNames: filterTags(req.AddNames, tags), + RemoveNamesOrTags: filterTags(req.RemoveNamesOrTags, tags), + } +} + +func filterTags(reqTags []string, filterTags *strset.Set) []string { + var filtered []string + for _, tag := range reqTags { + if filterTags.Has(tag) { + filtered = append(filtered, tag) + } + } + return filtered +} + +func tagsOrNamesThatTaskGroupRespondsTo(tasks []Task, requestTagsOrNames *strset.Set) (*strset.Set, *strset.Set) { + positiveRefs := strset.New() + for _, t := range tasks { + if sel, ok := t.(Selector); ok { + positiveRefs.Add("all") // everything responds to "all" + positiveRefs.Add(strset.Intersection(requestTagsOrNames, strset.New(sel.Selectors()...)).List()...) + } + positiveRefs.Add(t.Name()) + } + return positiveRefs, strset.Difference(requestTagsOrNames, positiveRefs) +} + +func allRequestReferences(s cataloging.SelectionRequest) *strset.Set { + st := strset.New() + st.Add(s.DefaultNamesOrTags...) + st.Add(s.SubSelectTags...) + st.Add(s.AddNames...) + st.Add(s.RemoveNamesOrTags...) + return st +} + // selectByExpressions the set of tasks to run based on the given expression(s). func selectByExpressions(ts tasks, nodes Expressions) (tasks, Selection) { if len(nodes) == 0 { diff --git a/internal/task/selection_test.go b/internal/task/selection_test.go index d834480df8d..c503a73d113 100644 --- a/internal/task/selection_test.go +++ b/internal/task/selection_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/anchore/syft/internal/sbomsync" - "github.com/anchore/syft/syft/cataloging/pkgcataloging" + "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" ) @@ -21,7 +21,7 @@ func dummyTask(name string, tags ...string) Task { } // note: this test fixture does not need to be kept up to date here, but makes a great test subject -func createDummyTasks() tasks { +func createDummyPackageTasks() tasks { return []Task{ // OS package installed catalogers dummyTask("alpm-db-cataloger", "directory", "installed", "image", "os", "alpm", "archlinux"), @@ -63,6 +63,15 @@ func createDummyTasks() tasks { } } +func createDummyFileTasks() tasks { + return []Task{ + dummyTask("file-content-cataloger", "file", "content"), + dummyTask("file-metadata-cataloger", "file", "metadata"), + dummyTask("file-digest-cataloger", "file", "digest"), + dummyTask("file-executable-cataloger", "file", "binary-metadata"), + } +} + func TestSelect(t *testing.T) { tests := []struct { @@ -72,7 +81,7 @@ func TestSelect(t *testing.T) { expressions []string wantNames []string wantTokens map[string]TokenSelection - wantRequest pkgcataloging.SelectionRequest + wantRequest cataloging.SelectionRequest wantErr assert.ErrorAssertionFunc }{ { @@ -82,11 +91,11 @@ func TestSelect(t *testing.T) { expressions: []string{}, wantNames: []string{}, wantTokens: map[string]TokenSelection{}, - wantRequest: pkgcataloging.SelectionRequest{}, + wantRequest: cataloging.SelectionRequest{}, }, { name: "use default tasks", - allTasks: createDummyTasks(), + allTasks: createDummyPackageTasks(), basis: []string{ "image", }, @@ -129,13 +138,13 @@ func TestSelect(t *testing.T) { "binary-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil), }, - wantRequest: pkgcataloging.SelectionRequest{ + wantRequest: cataloging.SelectionRequest{ DefaultNamesOrTags: []string{"image"}, }, }, { name: "select, add, and remove tasks", - allTasks: createDummyTasks(), + allTasks: createDummyPackageTasks(), basis: []string{ "image", }, @@ -175,7 +184,7 @@ func TestSelect(t *testing.T) { "binary-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil), }, - wantRequest: pkgcataloging.SelectionRequest{ + wantRequest: cataloging.SelectionRequest{ DefaultNamesOrTags: []string{"image"}, SubSelectTags: []string{"os"}, RemoveNamesOrTags: []string{"dpkg"}, @@ -184,7 +193,7 @@ func TestSelect(t *testing.T) { }, { name: "allow for partial selections", - allTasks: createDummyTasks(), + allTasks: createDummyPackageTasks(), basis: []string{ "image", }, @@ -228,7 +237,7 @@ func TestSelect(t *testing.T) { "binary-cataloger": newTokenSelection([]string{"image"}, nil), "sbom-cataloger": newTokenSelection([]string{"image"}, nil), }, - wantRequest: pkgcataloging.SelectionRequest{ + wantRequest: cataloging.SelectionRequest{ DefaultNamesOrTags: []string{"image"}, SubSelectTags: []string{"os", "rust-cargo-lock-cataloger"}, RemoveNamesOrTags: []string{"dpkg"}, @@ -238,7 +247,7 @@ func TestSelect(t *testing.T) { }, { name: "select all tasks", - allTasks: createDummyTasks(), + allTasks: createDummyPackageTasks(), basis: []string{ "all", }, @@ -299,13 +308,13 @@ func TestSelect(t *testing.T) { "github-action-workflow-usage-cataloger": newTokenSelection([]string{"all"}, nil), "sbom-cataloger": newTokenSelection([]string{"all"}, nil), }, - wantRequest: pkgcataloging.SelectionRequest{ + wantRequest: cataloging.SelectionRequest{ DefaultNamesOrTags: []string{"all"}, }, }, { name: "set default with multiple tags", - allTasks: createDummyTasks(), + allTasks: createDummyPackageTasks(), basis: []string{ "gemspec", "python", @@ -319,10 +328,31 @@ func TestSelect(t *testing.T) { "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"gemspec"}, nil), "python-installed-package-cataloger": newTokenSelection([]string{"python"}, nil), }, - wantRequest: pkgcataloging.SelectionRequest{ + wantRequest: cataloging.SelectionRequest{ DefaultNamesOrTags: []string{"gemspec", "python"}, }, }, + { + name: "automatically add file to default tags", + allTasks: createDummyFileTasks(), + basis: []string{}, + expressions: []string{}, + wantNames: []string{ + "file-content-cataloger", + "file-metadata-cataloger", + "file-digest-cataloger", + "file-executable-cataloger", + }, + wantTokens: map[string]TokenSelection{ + "file-content-cataloger": newTokenSelection([]string{"file"}, nil), + "file-metadata-cataloger": newTokenSelection([]string{"file"}, nil), + "file-digest-cataloger": newTokenSelection([]string{"file"}, nil), + "file-executable-cataloger": newTokenSelection([]string{"file"}, nil), + }, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"file"}, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -330,7 +360,7 @@ func TestSelect(t *testing.T) { tt.wantErr = assert.NoError } - req := pkgcataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) + req := cataloging.NewSelectionRequest().WithDefaults(tt.basis...).WithExpression(tt.expressions...) got, gotEvidence, err := Select(tt.allTasks, req) tt.wantErr(t, err) @@ -367,3 +397,303 @@ func TestSelect(t *testing.T) { }) } } + +func TestSelectInGroups(t *testing.T) { + tests := []struct { + name string + taskGroups [][]Task + selectionReq cataloging.SelectionRequest + wantGroups [][]string + wantTokens map[string]TokenSelection + wantRequest cataloging.SelectionRequest + wantErr assert.ErrorAssertionFunc + }{ + { + name: "select only within the file tasks (leave package tasks alone)", + taskGroups: [][]Task{ + createDummyPackageTasks(), + createDummyFileTasks(), + }, + selectionReq: cataloging.NewSelectionRequest(). + WithDefaults("image"). // note: file missing + WithSubSelections("content", "digest"), + wantGroups: [][]string{ + { + // this is the original, untouched package task list + "alpm-db-cataloger", + "apk-db-cataloger", + "dpkg-db-cataloger", + "portage-cataloger", + "rpm-db-cataloger", + "conan-info-cataloger", + "javascript-package-cataloger", + "php-composer-installed-cataloger", + "ruby-installed-gemspec-cataloger", + "rust-cargo-lock-cataloger", + "dotnet-portable-executable-cataloger", + "python-installed-package-cataloger", + "go-module-binary-cataloger", + "java-archive-cataloger", + "graalvm-native-image-cataloger", + "binary-cataloger", + "sbom-cataloger", + }, + { + // this has been filtered based on the request + "file-content-cataloger", + "file-digest-cataloger", + }, + }, + wantTokens: map[string]TokenSelection{ + // packages + "alpm-db-cataloger": newTokenSelection([]string{"image"}, nil), + "apk-db-cataloger": newTokenSelection([]string{"image"}, nil), + "binary-cataloger": newTokenSelection([]string{"image"}, nil), + "conan-info-cataloger": newTokenSelection([]string{"image"}, nil), + "dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil), + "dpkg-db-cataloger": newTokenSelection([]string{"image"}, nil), + "go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil), + "graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil), + "java-archive-cataloger": newTokenSelection([]string{"image"}, nil), + "javascript-package-cataloger": newTokenSelection([]string{"image"}, nil), + "php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil), + "portage-cataloger": newTokenSelection([]string{"image"}, nil), + "python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil), + "rpm-db-cataloger": newTokenSelection([]string{"image"}, nil), + "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil), + "rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil), + "sbom-cataloger": newTokenSelection([]string{"image"}, nil), + // files + "file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil), + "file-digest-cataloger": newTokenSelection([]string{"digest", "file"}, nil), + "file-executable-cataloger": newTokenSelection([]string{"file"}, nil), + "file-metadata-cataloger": newTokenSelection([]string{"file"}, nil), + }, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, // note: file automatically added + SubSelectTags: []string{"content", "digest"}, + }, + wantErr: assert.NoError, + }, + { + name: "select package tasks (leave file tasks alone)", + taskGroups: [][]Task{ + createDummyPackageTasks(), + createDummyFileTasks(), + }, + selectionReq: cataloging.NewSelectionRequest().WithDefaults("image").WithSubSelections("os"), + wantGroups: [][]string{ + { + // filtered based on the request + "alpm-db-cataloger", + "apk-db-cataloger", + "dpkg-db-cataloger", + "portage-cataloger", + "rpm-db-cataloger", + }, + { + // this is the original, untouched file task list + "file-content-cataloger", + "file-metadata-cataloger", + "file-digest-cataloger", + "file-executable-cataloger", + }, + }, + wantTokens: map[string]TokenSelection{ + // packages - os + "alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil), + "rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "portage-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + // packages - remaining + "binary-cataloger": newTokenSelection([]string{"image"}, nil), + "conan-info-cataloger": newTokenSelection([]string{"image"}, nil), + "dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil), + "go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil), + "graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil), + "java-archive-cataloger": newTokenSelection([]string{"image"}, nil), + "javascript-package-cataloger": newTokenSelection([]string{"image"}, nil), + "php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil), + "python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil), + "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil), + "rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil), + "sbom-cataloger": newTokenSelection([]string{"image"}, nil), + // files + "file-content-cataloger": newTokenSelection([]string{"file"}, nil), + "file-digest-cataloger": newTokenSelection([]string{"file"}, nil), + "file-executable-cataloger": newTokenSelection([]string{"file"}, nil), + "file-metadata-cataloger": newTokenSelection([]string{"file"}, nil), + }, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, + SubSelectTags: []string{"os"}, + }, + wantErr: assert.NoError, + }, + { + name: "select file and package tasks", + taskGroups: [][]Task{ + createDummyPackageTasks(), + createDummyFileTasks(), + }, + selectionReq: cataloging.NewSelectionRequest(). + WithDefaults("image"). + WithSubSelections("os", "content", "digest"), + wantGroups: [][]string{ + { + // filtered based on the request + "alpm-db-cataloger", + "apk-db-cataloger", + "dpkg-db-cataloger", + "portage-cataloger", + "rpm-db-cataloger", + }, + { + // filtered based on the request + "file-content-cataloger", + "file-digest-cataloger", + }, + }, + wantTokens: map[string]TokenSelection{ + // packages - os + "alpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "apk-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "rpm-archive-cataloger": newTokenSelection([]string{"os"}, nil), + "rpm-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "portage-cataloger": newTokenSelection([]string{"os", "image"}, nil), + "dpkg-db-cataloger": newTokenSelection([]string{"os", "image"}, nil), + // packages - remaining + "binary-cataloger": newTokenSelection([]string{"image"}, nil), + "conan-info-cataloger": newTokenSelection([]string{"image"}, nil), + "dotnet-portable-executable-cataloger": newTokenSelection([]string{"image"}, nil), + "go-module-binary-cataloger": newTokenSelection([]string{"image"}, nil), + "graalvm-native-image-cataloger": newTokenSelection([]string{"image"}, nil), + "java-archive-cataloger": newTokenSelection([]string{"image"}, nil), + "javascript-package-cataloger": newTokenSelection([]string{"image"}, nil), + "php-composer-installed-cataloger": newTokenSelection([]string{"image"}, nil), + "python-installed-package-cataloger": newTokenSelection([]string{"image"}, nil), + "ruby-installed-gemspec-cataloger": newTokenSelection([]string{"image"}, nil), + "rust-cargo-lock-cataloger": newTokenSelection([]string{"image"}, nil), + "sbom-cataloger": newTokenSelection([]string{"image"}, nil), + // files + "file-content-cataloger": newTokenSelection([]string{"file", "content"}, nil), // note extra tags + "file-digest-cataloger": newTokenSelection([]string{"file", "digest"}, nil), // note extra tags + "file-executable-cataloger": newTokenSelection([]string{"file"}, nil), + "file-metadata-cataloger": newTokenSelection([]string{"file"}, nil), + }, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, + SubSelectTags: []string{"os", "content", "digest"}, + }, + wantErr: assert.NoError, + }, + { + name: "complex selection with multiple operators across groups", + taskGroups: [][]Task{ + createDummyPackageTasks(), + createDummyFileTasks(), + }, + selectionReq: cataloging.NewSelectionRequest(). + WithDefaults("os"). // note: no file tag present + WithExpression("+github-actions-usage-cataloger", "-dpkg", "-digest", "content", "+file-metadata-cataloger", "-declared"), + wantGroups: [][]string{ + { + "alpm-db-cataloger", + "apk-db-cataloger", + "portage-cataloger", + "rpm-db-cataloger", + "github-actions-usage-cataloger", + }, + { + "file-content-cataloger", + "file-metadata-cataloger", + }, + }, + wantTokens: map[string]TokenSelection{ + // selected package tasks + "alpm-db-cataloger": newTokenSelection([]string{"os"}, nil), + "apk-db-cataloger": newTokenSelection([]string{"os"}, nil), + "dpkg-db-cataloger": newTokenSelection([]string{"os"}, []string{"dpkg"}), + "portage-cataloger": newTokenSelection([]string{"os"}, nil), + "rpm-archive-cataloger": newTokenSelection([]string{"os"}, []string{"declared"}), + "rpm-db-cataloger": newTokenSelection([]string{"os"}, nil), + "github-actions-usage-cataloger": newTokenSelection([]string{"github-actions-usage-cataloger"}, []string{"declared"}), + + // selected file tasks + "file-content-cataloger": newTokenSelection([]string{"content", "file"}, nil), + "file-metadata-cataloger": newTokenSelection([]string{"file-metadata-cataloger", "file"}, nil), + + // removed package tasks + "binary-cataloger": newTokenSelection(nil, []string{"declared"}), + "conan-cataloger": newTokenSelection(nil, []string{"declared"}), + "dart-pubspec-lock-cataloger": newTokenSelection(nil, []string{"declared"}), + "dotnet-deps-cataloger": newTokenSelection(nil, []string{"declared"}), + "elixir-mix-lock-cataloger": newTokenSelection(nil, []string{"declared"}), + "erlang-rebar-lock-cataloger": newTokenSelection(nil, []string{"declared"}), + "github-action-workflow-usage-cataloger": newTokenSelection(nil, []string{"declared"}), + "javascript-lock-cataloger": newTokenSelection(nil, []string{"declared"}), + "sbom-cataloger": newTokenSelection(nil, []string{"declared"}), + + // removed file tasks + "file-executable-cataloger": newTokenSelection([]string{"file"}, nil), + "file-digest-cataloger": newTokenSelection([]string{"file"}, []string{"digest"}), + }, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"os", "file"}, // note: file added automatically + SubSelectTags: []string{"content"}, + RemoveNamesOrTags: []string{"dpkg", "digest", "declared"}, + AddNames: []string{"github-actions-usage-cataloger", "file-metadata-cataloger"}, + }, + wantErr: assert.NoError, + }, + { + name: "invalid tag", + taskGroups: [][]Task{ + createDummyPackageTasks(), + createDummyFileTasks(), + }, + selectionReq: cataloging.NewSelectionRequest().WithDefaults("invalid"), + wantGroups: nil, + wantTokens: nil, + wantRequest: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"invalid", "file"}, + }, + wantErr: assert.Error, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = assert.NoError + } + + gotGroups, gotSelection, err := SelectInGroups(tt.taskGroups, tt.selectionReq) + tt.wantErr(t, err) + if err != nil { + // dev note: this is useful for debugging when needed... + //for _, e := range gotEvidence.Request.Expressions { + // t.Logf("expression (errors %q): %#v", e.Errors, e) + //} + + // note: we DON'T bail early in validations... this is because we should always return the full set of + // of selected tasks and surrounding evidence. + } + + var gotGroupNames [][]string + for _, group := range gotGroups { + var names []string + for _, task := range group { + names = append(names, task.Name()) + } + gotGroupNames = append(gotGroupNames, names) + } + + assert.Equal(t, tt.wantGroups, gotGroupNames) + assert.Equal(t, tt.wantTokens, gotSelection.TokensByTask) + assert.Equal(t, tt.wantRequest, gotSelection.Request) + }) + } +} diff --git a/syft/cataloging/filecataloging/tags.go b/syft/cataloging/filecataloging/tags.go new file mode 100644 index 00000000000..eced215c937 --- /dev/null +++ b/syft/cataloging/filecataloging/tags.go @@ -0,0 +1,6 @@ +package filecataloging + +const ( + // FileTag should be used to identify catalogers that are file-based. + FileTag = "file" +) diff --git a/syft/cataloging/pkgcataloging/selection.go b/syft/cataloging/pkgcataloging/selection.go new file mode 100644 index 00000000000..7d34211a600 --- /dev/null +++ b/syft/cataloging/pkgcataloging/selection.go @@ -0,0 +1,11 @@ +package pkgcataloging + +import ( + "github.com/anchore/syft/syft/cataloging" +) + +// SelectionRequest is deprecated: use cataloging.SelectionRequest instead +type SelectionRequest = cataloging.SelectionRequest + +// NewSelectionRequest is deprecated: use cataloging.NewSelectionRequest instead +var NewSelectionRequest = cataloging.NewSelectionRequest diff --git a/syft/cataloging/pkgcataloging/selection_request.go b/syft/cataloging/selection.go similarity index 97% rename from syft/cataloging/pkgcataloging/selection_request.go rename to syft/cataloging/selection.go index 7dc711e3ae7..b94da62fd38 100644 --- a/syft/cataloging/pkgcataloging/selection_request.go +++ b/syft/cataloging/selection.go @@ -1,8 +1,6 @@ -package pkgcataloging +package cataloging -import ( - "strings" -) +import "strings" type SelectionRequest struct { DefaultNamesOrTags []string `json:"default,omitempty"` diff --git a/syft/configuration_audit_trail.go b/syft/configuration_audit_trail.go index 78f43fece61..7346cefd38f 100644 --- a/syft/configuration_audit_trail.go +++ b/syft/configuration_audit_trail.go @@ -21,8 +21,8 @@ type configurationAuditTrail struct { } type catalogerManifest struct { - Requested pkgcataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"` - Used []string `json:"used" yaml:"used" mapstructure:"used"` + Requested cataloging.SelectionRequest `json:"requested" yaml:"requested" mapstructure:"requested"` + Used []string `json:"used" yaml:"used" mapstructure:"used"` } type marshalAPIConfiguration configurationAuditTrail diff --git a/syft/create_sbom.go b/syft/create_sbom.go index 1e526fd90c6..de732c5ad35 100644 --- a/syft/create_sbom.go +++ b/syft/create_sbom.go @@ -125,6 +125,9 @@ func monitorCatalogingTask(srcID artifact.ID, tasks [][]task.Task) *monitor.Cata func formatTaskNames(tasks []task.Task) []string { set := strset.New() for _, td := range tasks { + if td == nil { + continue + } set.Add(td.Name()) } list := set.List() diff --git a/syft/create_sbom_config.go b/syft/create_sbom_config.go index 8e9e76b5c3b..986722319ac 100644 --- a/syft/create_sbom_config.go +++ b/syft/create_sbom_config.go @@ -7,6 +7,7 @@ import ( "runtime/debug" "strings" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/internal/task" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/cataloging/filecataloging" @@ -27,14 +28,14 @@ type CreateSBOMConfig struct { Packages pkgcataloging.Config Files filecataloging.Config Parallelism int - CatalogerSelection pkgcataloging.SelectionRequest + CatalogerSelection cataloging.SelectionRequest // audit what tool is being used to generate the SBOM ToolName string ToolVersion string ToolConfiguration interface{} - packageTaskFactories task.PackageTaskFactories + packageTaskFactories task.Factories packageCatalogerReferences []pkgcataloging.CatalogerReference } @@ -149,7 +150,7 @@ func (c *CreateSBOMConfig) WithoutFiles() *CreateSBOMConfig { } // WithCatalogerSelection allows for adding to, removing from, or sub-selecting the final set of catalogers by name or tag. -func (c *CreateSBOMConfig) WithCatalogerSelection(selection pkgcataloging.SelectionRequest) *CreateSBOMConfig { +func (c *CreateSBOMConfig) WithCatalogerSelection(selection cataloging.SelectionRequest) *CreateSBOMConfig { c.CatalogerSelection = selection return c } @@ -165,6 +166,10 @@ func (c *CreateSBOMConfig) WithoutCatalogers() *CreateSBOMConfig { // WithCatalogers allows for adding user-provided catalogers to the final set of catalogers that will always be run // regardless of the source type or any cataloger selections provided. func (c *CreateSBOMConfig) WithCatalogers(catalogerRefs ...pkgcataloging.CatalogerReference) *CreateSBOMConfig { + for i := range catalogerRefs { + // ensure that all package catalogers have the package tag + catalogerRefs[i].Tags = append(catalogerRefs[i].Tags, pkgcataloging.PackageTag) + } c.packageCatalogerReferences = append(c.packageCatalogerReferences, catalogerRefs...) return c @@ -181,8 +186,8 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task environmentTasks := c.environmentTasks() relationshipsTasks := c.relationshipTasks(src) unknownTasks := c.unknownsTasks() - fileTasks := c.fileTasks() - pkgTasks, selectionEvidence, err := c.packageTasks(src) + + pkgTasks, fileTasks, selectionEvidence, err := c.selectTasks(src) if err != nil { return nil, nil, err } @@ -220,72 +225,90 @@ func (c *CreateSBOMConfig) makeTaskGroups(src source.Description) ([][]task.Task } // fileTasks returns the set of tasks that should be run to catalog files. -func (c *CreateSBOMConfig) fileTasks() []task.Task { - var tsks []task.Task - - if t := task.NewFileDigestCatalogerTask(c.Files.Selection, c.Files.Hashers...); t != nil { - tsks = append(tsks, t) - } - if t := task.NewFileMetadataCatalogerTask(c.Files.Selection); t != nil { - tsks = append(tsks, t) - } - if t := task.NewFileContentCatalogerTask(c.Files.Content); t != nil { - tsks = append(tsks, t) - } - if t := task.NewExecutableCatalogerTask(c.Files.Selection, c.Files.Executable); t != nil { - tsks = append(tsks, t) +func (c *CreateSBOMConfig) fileTasks(cfg task.CatalogingFactoryConfig) ([]task.Task, error) { + tsks, err := task.DefaultFileTaskFactories().Tasks(cfg) + if err != nil { + return nil, fmt.Errorf("unable to create file cataloger tasks: %w", err) } - return tsks + return tsks, nil } -// packageTasks returns the set of tasks that should be run to catalog packages. -func (c *CreateSBOMConfig) packageTasks(src source.Description) ([]task.Task, *task.Selection, error) { +// selectTasks returns the set of tasks that should be run to catalog packages and files. +func (c *CreateSBOMConfig) selectTasks(src source.Description) ([]task.Task, []task.Task, *task.Selection, error) { cfg := task.CatalogingFactoryConfig{ SearchConfig: c.Search, RelationshipsConfig: c.Relationships, DataGenerationConfig: c.DataGeneration, PackagesConfig: c.Packages, ComplianceConfig: c.Compliance, + FilesConfig: c.Files, } - persistentTasks, selectableTasks, err := c.allPackageTasks(cfg) + persistentPkgTasks, selectablePkgTasks, err := c.allPackageTasks(cfg) if err != nil { - return nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err) + return nil, nil, nil, fmt.Errorf("unable to create package cataloger tasks: %w", err) } - req, err := finalSelectionRequest(c.CatalogerSelection, src) + req, err := finalTaskSelectionRequest(c.CatalogerSelection, src) if err != nil { - return nil, nil, err + return nil, nil, nil, err } - finalTasks, selection, err := task.Select(selectableTasks, *req) + selectableFileTasks, err := c.fileTasks(cfg) if err != nil { - return nil, nil, err + return nil, nil, nil, err + } + + taskGroups := [][]task.Task{ + selectablePkgTasks, + selectableFileTasks, + } + + finalTaskGroups, selection, err := task.SelectInGroups(taskGroups, *req) + if err != nil { + return nil, nil, nil, err } - finalTasks = append(finalTasks, persistentTasks...) + finalPkgTasks := finalTaskGroups[0] + finalFileTasks := finalTaskGroups[1] - if len(finalTasks) == 0 { - return nil, nil, fmt.Errorf("no catalogers selected") + finalPkgTasks = append(finalPkgTasks, persistentPkgTasks...) + + if len(finalPkgTasks) == 0 && len(finalFileTasks) == 0 { + return nil, nil, nil, fmt.Errorf("no catalogers selected") } - return finalTasks, &selection, nil + logTaskNames(finalPkgTasks, "package cataloger") + logTaskNames(finalFileTasks, "file cataloger") + + return finalPkgTasks, finalFileTasks, &selection, nil } -func finalSelectionRequest(req pkgcataloging.SelectionRequest, src source.Description) (*pkgcataloging.SelectionRequest, error) { +func logTaskNames(tasks []task.Task, kind string) { + // log as tree output (like tree command) + log.Debugf("selected %d %s tasks", len(tasks), kind) + names := formatTaskNames(tasks) + for idx, t := range names { + if idx == len(tasks)-1 { + log.Tracef("└── %s", t) + } else { + log.Tracef("├── %s", t) + } + } +} + +func finalTaskSelectionRequest(req cataloging.SelectionRequest, src source.Description) (*cataloging.SelectionRequest, error) { if len(req.DefaultNamesOrTags) == 0 { - defaultTag, err := findDefaultTag(src) + defaultTags, err := findDefaultTags(src) if err != nil { return nil, fmt.Errorf("unable to determine default cataloger tag: %w", err) } - if defaultTag != "" { - req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTag) - } + req.DefaultNamesOrTags = append(req.DefaultNamesOrTags, defaultTags...) - req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTag, req.RemoveNamesOrTags) - req.SubSelectTags = replaceDefaultTagReferences(defaultTag, req.SubSelectTags) + req.RemoveNamesOrTags = replaceDefaultTagReferences(defaultTags, req.RemoveNamesOrTags) + req.SubSelectTags = replaceDefaultTagReferences(defaultTags, req.SubSelectTags) } return &req, nil @@ -377,21 +400,29 @@ func (c *CreateSBOMConfig) Create(ctx context.Context, src source.Source) (*sbom return CreateSBOM(ctx, src, c) } -func findDefaultTag(src source.Description) (string, error) { +func findDefaultTags(src source.Description) ([]string, error) { switch m := src.Metadata.(type) { case source.ImageMetadata: - return pkgcataloging.ImageTag, nil + return []string{pkgcataloging.ImageTag, filecataloging.FileTag}, nil case source.FileMetadata, source.DirectoryMetadata: - return pkgcataloging.DirectoryTag, nil + return []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, nil default: - return "", fmt.Errorf("unable to determine default cataloger tag for source type=%T", m) + return nil, fmt.Errorf("unable to determine default cataloger tag for source type=%T", m) } } -func replaceDefaultTagReferences(defaultTag string, lst []string) []string { +func replaceDefaultTagReferences(defaultTags []string, lst []string) []string { for i, tag := range lst { if strings.ToLower(tag) == "default" { - lst[i] = defaultTag + switch len(defaultTags) { + case 0: + lst[i] = "" + case 1: + lst[i] = defaultTags[0] + default: + // remove the default tag and add the individual tags + lst = append(lst[:i], append(defaultTags, lst[i+1:]...)...) + } } } return lst diff --git a/syft/create_sbom_config_test.go b/syft/create_sbom_config_test.go index 64a9a2a0af4..45afb5b9061 100644 --- a/syft/create_sbom_config_test.go +++ b/syft/create_sbom_config_test.go @@ -88,13 +88,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "image"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "image"), }, @@ -107,13 +107,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "directory"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"directory"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"directory", "file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "directory"), }, @@ -127,13 +127,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "directory"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"directory"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"directory", "file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "directory"), }, @@ -142,17 +142,18 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { { name: "no file digest cataloger", src: imgSrc, - cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithHashers()), + cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("digest")), wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "image"), - fileCatalogerNames(false, true, true), // note: the digest cataloger is not included + fileCatalogerNames("file-metadata", "content", "binary-metadata"), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, + RemoveNamesOrTags: []string{"digest"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "image"), }, @@ -161,17 +162,18 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { { name: "select no file catalogers", src: imgSrc, - cfg: DefaultCreateSBOMConfig().WithFilesConfig(filecataloging.DefaultConfig().WithSelection(file.NoFilesSelection)), + cfg: DefaultCreateSBOMConfig().WithCatalogerSelection(cataloging.NewSelectionRequest().WithRemovals("file")), wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "image"), - // note: there are no file catalogers in their own group + nil, // note: there is a file cataloging group, with no items in it relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, + RemoveNamesOrTags: []string{"file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "image"), }, @@ -186,14 +188,14 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { // note: there is a single group of catalogers for pkgs and files append( pkgCatalogerNamesWithTagOrName(t, "image"), - fileCatalogerNames(true, true, true)..., + fileCatalogerNames()..., ), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "image"), }, @@ -208,13 +210,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, }, Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "persistent"), }, @@ -229,13 +231,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"directory"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"directory", "file"}, }, Used: addTo(pkgCatalogerNamesWithTagOrName(t, "directory"), "persistent"), }, @@ -246,17 +248,17 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { src: imgSrc, cfg: DefaultCreateSBOMConfig().WithCatalogers( pkgcataloging.NewAlwaysEnabledCatalogerReference(newDummyCataloger("persistent")), - ).WithCatalogerSelection(pkgcataloging.NewSelectionRequest().WithSubSelections("javascript")), + ).WithCatalogerSelection(cataloging.NewSelectionRequest().WithSubSelections("javascript")), wantTaskNames: [][]string{ environmentCatalogerNames(), addTo(pkgIntersect("image", "javascript"), "persistent"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, SubSelectTags: []string{"javascript"}, }, Used: addTo(pkgIntersect("image", "javascript"), "persistent"), @@ -272,13 +274,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, }, Used: addTo(pkgCatalogerNamesWithTagOrName(t, "image"), "user-provided"), }, @@ -293,13 +295,13 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { wantTaskNames: [][]string{ environmentCatalogerNames(), pkgCatalogerNamesWithTagOrName(t, "image"), - fileCatalogerNames(true, true, true), + fileCatalogerNames(), relationshipCatalogerNames(), unknownsTaskNames(), }, wantManifest: &catalogerManifest{ - Requested: pkgcataloging.SelectionRequest{ - DefaultNamesOrTags: []string{"image"}, + Requested: cataloging.SelectionRequest{ + DefaultNamesOrTags: []string{"image", "file"}, }, Used: pkgCatalogerNamesWithTagOrName(t, "image"), }, @@ -314,9 +316,6 @@ func TestCreateSBOMConfig_makeTaskGroups(t *testing.T) { // sanity check require.NotEmpty(t, tt.wantTaskNames) - for _, group := range tt.wantTaskNames { - require.NotEmpty(t, group) - } // test the subject gotTasks, gotManifest, err := tt.cfg.makeTaskGroups(tt.src) @@ -378,17 +377,39 @@ func pkgCatalogerNamesWithTagOrName(t *testing.T, token string) []string { return names } -func fileCatalogerNames(digest, metadata, executable bool) []string { +func fileCatalogerNames(tokens ...string) []string { var names []string - if digest { - names = append(names, "file-digest-cataloger") - } - if executable { - names = append(names, "file-executable-cataloger") - } - if metadata { - names = append(names, "file-metadata-cataloger") + cfg := task.DefaultCatalogingFactoryConfig() +topLoop: + for _, factory := range task.DefaultFileTaskFactories() { + cat := factory(cfg) + + if cat == nil { + continue + } + + name := cat.Name() + + if len(tokens) == 0 { + names = append(names, name) + continue + } + + for _, token := range tokens { + if selector, ok := cat.(task.Selector); ok { + if selector.HasAllSelectors(token) { + names = append(names, name) + continue topLoop + } + + } + if name == token { + names = append(names, name) + } + } } + + sort.Strings(names) return names } @@ -436,7 +457,7 @@ func Test_replaceDefaultTagReferences(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, replaceDefaultTagReferences("replacement", tt.lst)) + assert.Equal(t, tt.want, replaceDefaultTagReferences([]string{"replacement"}, tt.lst)) }) } } @@ -446,7 +467,7 @@ func Test_findDefaultTag(t *testing.T) { tests := []struct { name string src source.Description - want string + want []string wantErr require.ErrorAssertionFunc }{ { @@ -454,21 +475,21 @@ func Test_findDefaultTag(t *testing.T) { src: source.Description{ Metadata: source.ImageMetadata{}, }, - want: pkgcataloging.ImageTag, + want: []string{pkgcataloging.ImageTag, filecataloging.FileTag}, }, { name: "directory", src: source.Description{ Metadata: source.DirectoryMetadata{}, }, - want: pkgcataloging.DirectoryTag, + want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, }, { name: "file", src: source.Description{ Metadata: source.FileMetadata{}, }, - want: pkgcataloging.DirectoryTag, // not a mistake... + want: []string{pkgcataloging.DirectoryTag, filecataloging.FileTag}, // not a mistake... }, { name: "unknown", @@ -483,7 +504,7 @@ func Test_findDefaultTag(t *testing.T) { if tt.wantErr == nil { tt.wantErr = require.NoError } - got, err := findDefaultTag(tt.src) + got, err := findDefaultTags(tt.src) tt.wantErr(t, err) if err != nil { return diff --git a/syft/internal/fileresolver/file_indexer.go b/syft/internal/fileresolver/file_indexer.go index fe110c4ced9..bc6d660ac6a 100644 --- a/syft/internal/fileresolver/file_indexer.go +++ b/syft/internal/fileresolver/file_indexer.go @@ -5,11 +5,12 @@ import ( "os" "path/filepath" + "github.com/wagoodman/go-progress" + "github.com/anchore/stereoscope/pkg/file" "github.com/anchore/stereoscope/pkg/filetree" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/internal/windows" - "github.com/wagoodman/go-progress" ) type fileIndexer struct { diff --git a/syft/internal/fileresolver/file_indexer_test.go b/syft/internal/fileresolver/file_indexer_test.go index cce3981382d..165bc39d022 100644 --- a/syft/internal/fileresolver/file_indexer_test.go +++ b/syft/internal/fileresolver/file_indexer_test.go @@ -1,13 +1,15 @@ package fileresolver import ( - "github.com/anchore/stereoscope/pkg/file" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "io/fs" "os" "path" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/stereoscope/pkg/file" ) // - Verify that both the parent and the path are indexed diff --git a/syft/internal/fileresolver/filetree_resolver_test.go b/syft/internal/fileresolver/filetree_resolver_test.go index e385ec8501f..1a3d661722d 100644 --- a/syft/internal/fileresolver/filetree_resolver_test.go +++ b/syft/internal/fileresolver/filetree_resolver_test.go @@ -14,13 +14,14 @@ import ( "testing" "time" - stereoscopeFile "github.com/anchore/stereoscope/pkg/file" - "github.com/anchore/syft/syft/file" "github.com/google/go-cmp/cmp" "github.com/scylladb/go-set/strset" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/goleak" + + stereoscopeFile "github.com/anchore/stereoscope/pkg/file" + "github.com/anchore/syft/syft/file" ) // Tests for filetree resolver when directory is used for index