From 14b1a8337eb1d7d6f6829e10d60e3b589aca0732 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 12 Sep 2024 11:50:20 -0400 Subject: [PATCH 1/2] feat: --enrich flag to enable data enrichment Signed-off-by: Keith Zantow --- .golangci.yaml | 2 +- cmd/grype/cli/commands/root.go | 25 +------ cmd/grype/cli/options/datasources.go | 19 +---- cmd/grype/cli/options/grype.go | 106 +++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 40 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 184f0f8ec2d..07d91a664a5 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -12,10 +12,10 @@ linters: enable: - asciicheck - bodyclose + - copyloopvar - dogsled - dupl - errcheck - - exportloopref - funlen - gocognit - goconst diff --git a/cmd/grype/cli/commands/root.go b/cmd/grype/cli/commands/root.go index f23382db24a..5649abd5b4b 100644 --- a/cmd/grype/cli/commands/root.go +++ b/cmd/grype/cli/commands/root.go @@ -34,7 +34,6 @@ import ( "github.com/anchore/grype/internal/format" "github.com/anchore/grype/internal/log" "github.com/anchore/grype/internal/stringutil" - "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/linux" syftPkg "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/sbom" @@ -159,7 +158,7 @@ func runGrype(app clio.Application, opts *options.Grype, userInput string) (errs // the SBOM is returned for downstream formatting concerns // grype uses the SBOM in combination with syft formatters to produce cycloneDX // with vulnerability information appended - packages, pkgContext, s, err = pkg.Provide(userInput, getProviderConfig(opts)) + packages, pkgContext, s, err = pkg.Provide(userInput, opts.ToProviderConfig()) if err != nil { return fmt.Errorf("failed to catalog: %w", err) } @@ -282,7 +281,7 @@ func getMatchers(opts *options.Grype) []matcher.Matcher { return matcher.NewDefaultMatchers( matcher.Config{ Java: java.MatcherConfig{ - ExternalSearchConfig: opts.ExternalSources.ToJavaMatcherConfig(), + ExternalSearchConfig: opts.ToJavaExternalSearchConfig(), UseCPEs: opts.Match.Java.UseCPEs, }, Ruby: ruby.MatcherConfig(opts.Match.Ruby), @@ -299,26 +298,6 @@ func getMatchers(opts *options.Grype) []matcher.Matcher { ) } -func getProviderConfig(opts *options.Grype) pkg.ProviderConfig { - cfg := syft.DefaultCreateSBOMConfig() - cfg.Packages.JavaArchive.IncludeIndexedArchives = opts.Search.IncludeIndexedArchives - cfg.Packages.JavaArchive.IncludeUnindexedArchives = opts.Search.IncludeUnindexedArchives - - return pkg.ProviderConfig{ - SyftProviderConfig: pkg.SyftProviderConfig{ - RegistryOptions: opts.Registry.ToOptions(), - Exclusions: opts.Exclusions, - SBOMOptions: cfg, - Platform: opts.Platform, - Name: opts.Name, - DefaultImagePullSource: opts.DefaultImagePullSource, - }, - SynthesisConfig: pkg.SynthesisConfig{ - GenerateMissingCPEs: opts.GenerateMissingCPEs, - }, - } -} - func validateDBLoad(loadErr error, status *db.Status) error { if loadErr != nil { return fmt.Errorf("failed to load vulnerability db: %w", loadErr) diff --git a/cmd/grype/cli/options/datasources.go b/cmd/grype/cli/options/datasources.go index 987e0f76ff9..0d9eaa52c6c 100644 --- a/cmd/grype/cli/options/datasources.go +++ b/cmd/grype/cli/options/datasources.go @@ -2,7 +2,6 @@ package options import ( "github.com/anchore/clio" - "github.com/anchore/grype/grype/matcher/java" ) const ( @@ -10,7 +9,7 @@ const ( ) type externalSources struct { - Enable bool `yaml:"enable" json:"enable" mapstructure:"enable"` + Enable *bool `yaml:"enable" json:"enable" mapstructure:"enable"` Maven maven `yaml:"maven" json:"maven" mapstructure:"maven"` } @@ -19,31 +18,19 @@ var _ interface { } = (*externalSources)(nil) type maven struct { - SearchUpstreamBySha1 bool `yaml:"search-upstream" json:"searchUpstreamBySha1" mapstructure:"search-maven-upstream"` + SearchUpstreamBySha1 *bool `yaml:"search-upstream" json:"searchUpstreamBySha1" mapstructure:"search-maven-upstream"` BaseURL string `yaml:"base-url" json:"baseUrl" mapstructure:"base-url"` } func defaultExternalSources() externalSources { return externalSources{ Maven: maven{ - SearchUpstreamBySha1: true, + SearchUpstreamBySha1: nil, BaseURL: defaultMavenBaseURL, }, } } -func (cfg externalSources) ToJavaMatcherConfig() java.ExternalSearchConfig { - // always respect if global config is disabled - smu := cfg.Maven.SearchUpstreamBySha1 - if !cfg.Enable { - smu = cfg.Enable - } - return java.ExternalSearchConfig{ - SearchMavenUpstream: smu, - MavenBaseURL: cfg.Maven.BaseURL, - } -} - func (cfg *externalSources) DescribeFields(descriptions clio.FieldDescriptionSet) { descriptions.Add(&cfg.Enable, `enable Grype searching network source for additional information`) descriptions.Add(&cfg.Maven.SearchUpstreamBySha1, `search for Maven artifacts by SHA1`) diff --git a/cmd/grype/cli/options/grype.go b/cmd/grype/cli/options/grype.go index ca822d4b1c0..0b8bf55ea71 100644 --- a/cmd/grype/cli/options/grype.go +++ b/cmd/grype/cli/options/grype.go @@ -2,11 +2,16 @@ package options import ( "fmt" + "strings" "github.com/anchore/clio" "github.com/anchore/grype/grype/match" + "github.com/anchore/grype/grype/matcher/java" + "github.com/anchore/grype/grype/pkg" "github.com/anchore/grype/grype/vulnerability" "github.com/anchore/grype/internal/format" + "github.com/anchore/grype/internal/log" + "github.com/anchore/syft/syft" "github.com/anchore/syft/syft/source" ) @@ -25,6 +30,7 @@ type Grype struct { Ignore []match.IgnoreRule `yaml:"ignore" json:"ignore" mapstructure:"ignore"` Exclusions []string `yaml:"exclude" json:"exclude" mapstructure:"exclude"` DB Database `yaml:"db" json:"db" mapstructure:"db"` + Enrich []string `yaml:"enrich" json:"enrich" mapstructure:"enrich"` ExternalSources externalSources `yaml:"external-sources" json:"externalSources" mapstructure:"external-sources"` Match matchConfig `yaml:"match" json:"match" mapstructure:"match"` FailOn string `yaml:"fail-on-severity" json:"fail-on-severity" mapstructure:"fail-on-severity"` @@ -136,6 +142,9 @@ func (o *Grype) AddFlags(flags clio.FlagSet) { "vex", "", "a list of VEX documents to consider when producing scanning results", ) + + flags.StringArrayVarP(&o.Enrich, "enrich", "", + fmt.Sprintf("enable package data enrichment from local and online sources (options: %s)", strings.Join(publicisedEnrichmentOptions, ", "))) } func (o *Grype) PostLoad() error { @@ -183,9 +192,106 @@ VEX fields apply when Grype reads vex data: `) descriptions.Add(&o.VexAdd, `VEX statuses to consider as ignored rules`) descriptions.Add(&o.MatchUpstreamKernelHeaders, `match kernel-header packages with upstream kernel as kernel vulnerabilities`) + + descriptions.Add(&o.Enrich, fmt.Sprintf(`Enable data enrichment operations, which can utilize services such as Maven Central and NPM. +Use: all to enable everything. Available options are: %s`, strings.Join(publicisedEnrichmentOptions, ", "))) } func (o Grype) FailOnSeverity() *vulnerability.Severity { severity := vulnerability.ParseSeverity(o.FailOn) return &severity } + +func (o *Grype) ToProviderConfig() pkg.ProviderConfig { + cfg := syft.DefaultCreateSBOMConfig() + cfg.Packages.JavaArchive.IncludeIndexedArchives = o.Search.IncludeIndexedArchives + cfg.Packages.JavaArchive.IncludeUnindexedArchives = o.Search.IncludeUnindexedArchives + cfg = cfg.WithPackagesConfig(cfg.Packages. + WithJavaArchiveConfig(cfg.Packages.JavaArchive. + WithUseNetwork(*multiLevelOption(false, enrichmentEnabled(o.Enrich, "java", "maven"))), + )) + + return pkg.ProviderConfig{ + SyftProviderConfig: pkg.SyftProviderConfig{ + RegistryOptions: o.Registry.ToOptions(), + Exclusions: o.Exclusions, + SBOMOptions: cfg, + Platform: o.Platform, + Name: o.Name, + DefaultImagePullSource: o.DefaultImagePullSource, + }, + SynthesisConfig: pkg.SynthesisConfig{ + GenerateMissingCPEs: o.GenerateMissingCPEs, + }, + } +} + +func (o Grype) ToJavaExternalSearchConfig() java.ExternalSearchConfig { + // always respect if global config is disabled + return java.ExternalSearchConfig{ + SearchMavenUpstream: *multiLevelOption(false, enrichmentEnabled(o.Enrich, "java", "maven"), o.ExternalSources.Enable, o.ExternalSources.Maven.SearchUpstreamBySha1), + MavenBaseURL: o.ExternalSources.Maven.BaseURL, + } +} + +func multiLevelOption[T any](defaultValue T, option ...*T) *T { + result := defaultValue + for _, opt := range option { + if opt != nil { + result = *opt + } + } + return &result +} + +var publicisedEnrichmentOptions = []string{ + "all", + "java", +} + +func enrichmentEnabled(enrichDirectives []string, features ...string) *bool { + if len(enrichDirectives) == 0 { + return nil + } + + enabled := func(features ...string) *bool { + for _, directive := range enrichDirectives { + enable := true + directive = strings.TrimPrefix(directive, "+") // +java and java are equivalent + if strings.HasPrefix(directive, "-") { + directive = directive[1:] + enable = false + } + for _, feature := range features { + if directive == feature { + return &enable + } + } + } + return nil + } + + enableAll := enabled("all") + disableAll := enabled("none") + + if disableAll != nil && *disableAll { + if enableAll != nil { + log.Warn("you have specified to both enable and disable all enrichment functionality, defaulting to disabled") + } + enableAll = ptr(false) + } + + // check for explicit enable/disable of feature names + for _, feat := range features { + enableFeature := enabled(feat) + if enableFeature != nil { + return enableFeature + } + } + + return enableAll +} + +func ptr[T any](val T) *T { + return &val +} From a7f9e30607f6cefbaad4b725393d7e442a60b8da Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Thu, 12 Sep 2024 11:56:47 -0400 Subject: [PATCH 2/2] chore: update test Signed-off-by: Keith Zantow --- cmd/grype/cli/commands/root_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/grype/cli/commands/root_test.go b/cmd/grype/cli/commands/root_test.go index 82ed09b706c..bcc76f2c70e 100644 --- a/cmd/grype/cli/commands/root_test.go +++ b/cmd/grype/cli/commands/root_test.go @@ -75,8 +75,8 @@ func Test_getProviderConfig(t *testing.T) { cmpopts.IgnoreFields(binary.Classifier{}, "EvidenceMatcher"), cmpopts.IgnoreUnexported(syft.CreateSBOMConfig{}), } - if d := cmp.Diff(tt.want, getProviderConfig(tt.opts), opts...); d != "" { - t.Errorf("getProviderConfig() mismatch (-want +got):\n%s", d) + if d := cmp.Diff(tt.want, tt.opts.ToProviderConfig(), opts...); d != "" { + t.Errorf("opts.ToProviderConfig() mismatch (-want +got):\n%s", d) } }) }