diff --git a/pkg/vulnloader/nvdloader/convert.go b/pkg/vulnloader/nvdloader/convert.go new file mode 100644 index 000000000..05bc52c20 --- /dev/null +++ b/pkg/vulnloader/nvdloader/convert.go @@ -0,0 +1,349 @@ +package nvdloader + +import ( + "fmt" + "strings" + "time" + + apischema "github.com/facebookincubator/nvdtools/cveapi/nvd/schema" + jsonschema "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema" +) + +const ( + apiTimeFormat = "2006-01-02T15:04:05.999" + jsonTimeFormat = "2006-01-02T15:04Z" +) + +func toJSON(vulns []*apischema.CVEAPIJSON20DefCVEItem) ([]*jsonschema.NVDCVEFeedJSON10DefCVEItem, error) { + if vulns == nil { + return nil, nil + } + + cveItems := make([]*jsonschema.NVDCVEFeedJSON10DefCVEItem, 0, len(vulns)) + for _, vuln := range vulns { + if vuln.CVE == nil { + continue + } + + // Ignore vulnerabilities older than 2002, as the JSON feeds only had >= 2002. + parts := strings.Split(vuln.CVE.ID, "-") + if len(parts) != 3 || parts[1] < "2002" { + continue + } + + cve := vuln.CVE + + modifiedTime, err := toTime(cve.LastModified) + if err != nil { + return nil, fmt.Errorf("converting LastModified for %s: %w", cve.ID, err) + } + publishedTime, err := toTime(cve.Published) + if err != nil { + return nil, fmt.Errorf("converting Published for %s: %w", cve.ID, err) + } + + impact, err := toImpact(cve.Metrics) + if err != nil { + return nil, fmt.Errorf("converting Impact for %s: %w", cve.ID, err) + } + + cveItems = append(cveItems, &jsonschema.NVDCVEFeedJSON10DefCVEItem{ + CVE: toCVE(cve), + Configurations: toConfigurations(cve.Configurations), + Impact: impact, + LastModifiedDate: modifiedTime, + PublishedDate: publishedTime, + }) + } + + return cveItems, nil +} + +// It is up to the caller to ensure cve is not nil. +func toCVE(cve *apischema.CVEAPIJSON20CVEItem) *jsonschema.CVEJSON40 { + descriptions := make([]*jsonschema.CVEJSON40LangString, 0, 1) + for _, description := range cve.Descriptions { + // Only keep the English description. + if description.Lang != "en" { + continue + } + + descriptions = append(descriptions, &jsonschema.CVEJSON40LangString{ + Lang: description.Lang, + Value: description.Value, + }) + } + + return &jsonschema.CVEJSON40{ + CVEDataMeta: &jsonschema.CVEJSON40CVEDataMeta{ + ID: cve.ID, + }, + Description: &jsonschema.CVEJSON40Description{ + DescriptionData: descriptions, + }, + } +} + +func toImpact(metrics *apischema.CVEAPIJSON20CVEItemMetrics) (*jsonschema.NVDCVEFeedJSON10DefImpact, error) { + // Impact is allowed to be empty. + if metrics == nil { + return new(jsonschema.NVDCVEFeedJSON10DefImpact), nil + } + + // It is possible and allowed for one or even both of these to be empty. + metricV2 := toBaseMetricV2(metrics.CvssMetricV2) + metricV3 := toBaseMetricV3(metrics.CvssMetricV30, metrics.CvssMetricV31) + return &jsonschema.NVDCVEFeedJSON10DefImpact{ + BaseMetricV2: metricV2, + BaseMetricV3: metricV3, + }, nil +} + +func toBaseMetricV2(metrics []*apischema.CVEAPIJSON20CVSSV2) *jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV2 { + if len(metrics) == 0 { + return nil + } + + var cvss *apischema.CVEAPIJSON20CVSSV2 + var cvssData *apischema.CVSSV20 + for _, metric := range metrics { + if metric.Type == "Primary" { + cvss = metric + cvssData = metric.CvssData + break + } + } + // 1.1 JSON feeds only serve the "Primary" (NVD) CVSS score. + if cvss == nil { + return nil + } + + return &jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV2{ + AcInsufInfo: cvss.AcInsufInfo, + CVSSV2: &jsonschema.CVSSV20{ + AccessComplexity: cvssData.AccessComplexity, + AccessVector: cvssData.AccessVector, + Authentication: cvssData.Authentication, + AvailabilityImpact: cvssData.AvailabilityImpact, + AvailabilityRequirement: cvssData.AvailabilityRequirement, + BaseScore: cvssData.BaseScore, + CollateralDamagePotential: cvssData.CollateralDamagePotential, + ConfidentialityImpact: cvssData.ConfidentialityImpact, + ConfidentialityRequirement: cvssData.ConfidentialityRequirement, + EnvironmentalScore: cvssData.EnvironmentalScore, + Exploitability: cvssData.Exploitability, + IntegrityImpact: cvssData.IntegrityImpact, + IntegrityRequirement: cvssData.IntegrityRequirement, + RemediationLevel: cvssData.RemediationLevel, + ReportConfidence: cvssData.ReportConfidence, + TargetDistribution: cvssData.TargetDistribution, + TemporalScore: cvssData.TemporalScore, + VectorString: cvssData.VectorString, + Version: cvssData.Version, + }, + ExploitabilityScore: cvss.ExploitabilityScore, + ImpactScore: cvss.ImpactScore, + ObtainAllPrivilege: cvss.ObtainAllPrivilege, + ObtainOtherPrivilege: cvss.ObtainOtherPrivilege, + ObtainUserPrivilege: cvss.ObtainUserPrivilege, + Severity: cvss.BaseSeverity, + UserInteractionRequired: cvss.UserInteractionRequired, + } +} + +func toBaseMetricV3(metrics30 []*apischema.CVEAPIJSON20CVSSV30, metrics31 []*apischema.CVEAPIJSON20CVSSV31) *jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV3 { + switch { + case len(metrics31) != 0: + return toBaseMetricV31(metrics31) + case len(metrics30) != 0: + return toBaseMetricV30(metrics30) + default: + return nil + } +} + +func toBaseMetricV31(metrics []*apischema.CVEAPIJSON20CVSSV31) *jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV3 { + var cvss *apischema.CVEAPIJSON20CVSSV31 + var cvssData *apischema.CVSSV31 + for _, metric := range metrics { + if metric.Type == "Primary" { + cvss = metric + cvssData = metric.CvssData + break + } + } + // 1.1 JSON feeds only serve the "Primary" (NVD) CVSS score. + if cvss == nil { + return nil + } + + return &jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV3{ + CVSSV3: &jsonschema.CVSSV30{ + AttackComplexity: cvssData.AttackComplexity, + AttackVector: cvssData.AttackVector, + AvailabilityImpact: cvssData.AvailabilityImpact, + AvailabilityRequirement: cvssData.AvailabilityRequirement, + BaseScore: cvssData.BaseScore, + BaseSeverity: cvssData.BaseSeverity, + ConfidentialityImpact: cvssData.ConfidentialityImpact, + ConfidentialityRequirement: cvssData.ConfidentialityRequirement, + EnvironmentalScore: cvssData.EnvironmentalScore, + EnvironmentalSeverity: cvssData.EnvironmentalSeverity, + ExploitCodeMaturity: cvssData.ExploitCodeMaturity, + IntegrityImpact: cvssData.IntegrityImpact, + IntegrityRequirement: cvssData.IntegrityRequirement, + ModifiedAttackComplexity: cvssData.ModifiedAttackComplexity, + ModifiedAttackVector: cvssData.ModifiedAttackVector, + ModifiedAvailabilityImpact: cvssData.ModifiedAvailabilityImpact, + ModifiedConfidentialityImpact: cvssData.ModifiedConfidentialityImpact, + ModifiedIntegrityImpact: cvssData.ModifiedIntegrityImpact, + ModifiedPrivilegesRequired: cvssData.ModifiedPrivilegesRequired, + ModifiedScope: cvssData.ModifiedScope, + ModifiedUserInteraction: cvssData.ModifiedUserInteraction, + PrivilegesRequired: cvssData.PrivilegesRequired, + RemediationLevel: cvssData.RemediationLevel, + ReportConfidence: cvssData.ReportConfidence, + Scope: cvssData.Scope, + TemporalScore: cvssData.TemporalScore, + TemporalSeverity: cvssData.TemporalSeverity, + UserInteraction: cvssData.UserInteraction, + VectorString: cvssData.VectorString, + Version: cvssData.Version, + }, + ExploitabilityScore: cvss.ExploitabilityScore, + ImpactScore: cvss.ImpactScore, + } +} + +func toBaseMetricV30(metrics []*apischema.CVEAPIJSON20CVSSV30) *jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV3 { + var cvss *apischema.CVEAPIJSON20CVSSV30 + var cvssData *apischema.CVSSV30 + for _, metric := range metrics { + if metric.Type == "Primary" { + cvss = metric + cvssData = metric.CvssData + break + } + } + // 1.1 JSON feeds only serve the "Primary" (NVD) CVSS score. + if cvss == nil { + return nil + } + + return &jsonschema.NVDCVEFeedJSON10DefImpactBaseMetricV3{ + CVSSV3: &jsonschema.CVSSV30{ + AttackComplexity: cvssData.AttackComplexity, + AttackVector: cvssData.AttackVector, + AvailabilityImpact: cvssData.AvailabilityImpact, + AvailabilityRequirement: cvssData.AvailabilityRequirement, + BaseScore: cvssData.BaseScore, + BaseSeverity: cvssData.BaseSeverity, + ConfidentialityImpact: cvssData.ConfidentialityImpact, + ConfidentialityRequirement: cvssData.ConfidentialityRequirement, + EnvironmentalScore: cvssData.EnvironmentalScore, + EnvironmentalSeverity: cvssData.EnvironmentalSeverity, + ExploitCodeMaturity: cvssData.ExploitCodeMaturity, + IntegrityImpact: cvssData.IntegrityImpact, + IntegrityRequirement: cvssData.IntegrityRequirement, + ModifiedAttackComplexity: cvssData.ModifiedAttackComplexity, + ModifiedAttackVector: cvssData.ModifiedAttackVector, + ModifiedAvailabilityImpact: cvssData.ModifiedAvailabilityImpact, + ModifiedConfidentialityImpact: cvssData.ModifiedConfidentialityImpact, + ModifiedIntegrityImpact: cvssData.ModifiedIntegrityImpact, + ModifiedPrivilegesRequired: cvssData.ModifiedPrivilegesRequired, + ModifiedScope: cvssData.ModifiedScope, + ModifiedUserInteraction: cvssData.ModifiedUserInteraction, + PrivilegesRequired: cvssData.PrivilegesRequired, + RemediationLevel: cvssData.RemediationLevel, + ReportConfidence: cvssData.ReportConfidence, + Scope: cvssData.Scope, + TemporalScore: cvssData.TemporalScore, + TemporalSeverity: cvssData.TemporalSeverity, + UserInteraction: cvssData.UserInteraction, + VectorString: cvssData.VectorString, + Version: cvssData.Version, + }, + ExploitabilityScore: cvss.ExploitabilityScore, + ImpactScore: cvss.ImpactScore, + } +} + +func toConfigurations(configs []*apischema.CVEAPIJSON20Config) *jsonschema.NVDCVEFeedJSON10DefConfigurations { + // Configurations is allowed to be empty. + if len(configs) == 0 { + return new(jsonschema.NVDCVEFeedJSON10DefConfigurations) + } + + jsonConfigs := &jsonschema.NVDCVEFeedJSON10DefConfigurations{ + Nodes: make([]*jsonschema.NVDCVEFeedJSON10DefNode, 0, len(configs)), + } + for _, config := range configs { + jsonConfigs.Nodes = append(jsonConfigs.Nodes, toNode(config)) + } + + return jsonConfigs +} + +func toNode(config *apischema.CVEAPIJSON20Config) *jsonschema.NVDCVEFeedJSON10DefNode { + // If there is only one node, then just create a single JSON node + // using the API node's attributes. + if len(config.Nodes) == 1 { + node := config.Nodes[0] + return &jsonschema.NVDCVEFeedJSON10DefNode{ + CPEMatch: toCPEMatch(node), + Negate: node.Negate, + Operator: node.Operator, + } + } + + // The v2 API schema only makes it seem like there can be a single level of children. + // I do not know if this holds true in practice in the 1.1 schema. + // The samples I have checked seem to only have a single level of children, + // and the fact the new schema only allows for a single level tells me + // this is probably correct. + children := make([]*jsonschema.NVDCVEFeedJSON10DefNode, 0, len(config.Nodes)) + for _, node := range config.Nodes { + children = append(children, &jsonschema.NVDCVEFeedJSON10DefNode{ + CPEMatch: toCPEMatch(node), + Negate: node.Negate, + Operator: node.Operator, + }) + } + + return &jsonschema.NVDCVEFeedJSON10DefNode{ + Children: children, + Negate: config.Negate, + Operator: config.Operator, + } +} + +func toCPEMatch(node *apischema.CVEAPIJSON20Node) []*jsonschema.NVDCVEFeedJSON10DefCPEMatch { + cpeMatch := make([]*jsonschema.NVDCVEFeedJSON10DefCPEMatch, 0, len(node.CpeMatch)) + for _, cpe := range node.CpeMatch { + jsonCPEMatch := &jsonschema.NVDCVEFeedJSON10DefCPEMatch{ + VersionEndExcluding: cpe.VersionEndExcluding, + VersionEndIncluding: cpe.VersionEndIncluding, + VersionStartExcluding: cpe.VersionStartExcluding, + VersionStartIncluding: cpe.VersionStartIncluding, + Vulnerable: cpe.Vulnerable, + } + if strings.HasPrefix(cpe.Criteria, "cpe:2.3") { + jsonCPEMatch.Cpe23Uri = cpe.Criteria + } else { + jsonCPEMatch.Cpe22Uri = cpe.Criteria + } + + cpeMatch = append(cpeMatch, jsonCPEMatch) + } + + return cpeMatch +} + +func toTime(t string) (string, error) { + apiTime, err := time.Parse(apiTimeFormat, t) + if err != nil { + return "", err + } + + return apiTime.Format(jsonTimeFormat), nil +} diff --git a/pkg/vulnloader/nvdloader/loader.go b/pkg/vulnloader/nvdloader/loader.go index 7c203a65e..e79d97c6a 100644 --- a/pkg/vulnloader/nvdloader/loader.go +++ b/pkg/vulnloader/nvdloader/loader.go @@ -1,7 +1,6 @@ package nvdloader import ( - "compress/gzip" "encoding/json" "fmt" "net/http" @@ -9,9 +8,9 @@ import ( "path/filepath" "time" - "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema" + apischema "github.com/facebookincubator/nvdtools/cveapi/nvd/schema" + jsonschema "github.com/facebookincubator/nvdtools/cvefeed/nvd/schema" "github.com/facebookincubator/nvdtools/wfn" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/stackrox/rox/pkg/httputil/proxy" "github.com/stackrox/rox/pkg/utils" @@ -19,17 +18,19 @@ import ( "github.com/stackrox/scanner/pkg/vulnloader" ) -var ( - client = http.Client{ - Timeout: 2 * time.Minute, - Transport: proxy.RoundTripper(), - } -) +const urlFmt = `https://services.nvd.nist.gov/rest/json/cves/2.0?noRejected&startIndex=%d` + +var client = http.Client{ + Timeout: 2 * time.Minute, + Transport: proxy.RoundTripper(), +} func init() { vulnloader.RegisterLoader(vulndump.NVDDirName, &loader{}) } +var _ vulnloader.Loader = (*loader)(nil) + type loader struct{} // DownloadFeedsToPath downloads the NVD feeds to the given path. @@ -37,103 +38,177 @@ type loader struct{} // one json file for each year of NVD data. func (l *loader) DownloadFeedsToPath(outputDir string) error { // Fetch NVD enrichment data from curated repos - enrichmentMap, err := Fetch() + enrichments, err := Fetch() if err != nil { - return errors.Wrap(err, "could not fetch NVD enrichment sources") + return fmt.Errorf("could not fetch NVD enrichment sources: %w", err) } nvdDir := filepath.Join(outputDir, vulndump.NVDDirName) if err := os.MkdirAll(nvdDir, 0755); err != nil { - return errors.Wrapf(err, "creating subdir for %s", vulndump.NVDDirName) + return fmt.Errorf("creating subdir for %s: %w", vulndump.NVDDirName, err) } - endYear := time.Now().Year() - for year := 2002; year <= endYear; year++ { - if err := downloadFeedForYear(enrichmentMap, nvdDir, year); err != nil { - return err - } + + var fileNo, totalVulns int + + // Explicitly set startIdx to parallel how this is all done within the loop below. + startIdx := 0 + apiResp, err := query(fmt.Sprintf(urlFmt, startIdx)) + if err != nil { + return err } - return nil -} + var i int + // Buffer to store vulns until they are written to a file. + cveItems := make([]*jsonschema.NVDCVEFeedJSON10DefCVEItem, 0, 20_000) + for apiResp.ResultsPerPage != 0 { + vulns, err := toJSON(apiResp.Vulnerabilities) + if err != nil { + return fmt.Errorf("failed to convert API vulns to JSON: %w", err) + } -func removeInvalidCPEs(item *schema.NVDCVEFeedJSON10DefNode) { - cpeMatches := item.CPEMatch[:0] - for _, cpeMatch := range item.CPEMatch { - if cpeMatch.Cpe23Uri == "" { - cpeMatches = append(cpeMatches, cpeMatch) - continue + if len(vulns) != 0 { + cveItems = append(cveItems, vulns...) + + i++ + // Write to disk every ~20,000 vulnerabilities. + if i == 10 { + i = 0 + + enrichCVEItems(&cveItems, enrichments) + + feed := &jsonschema.NVDCVEFeedJSON10{ + CVEItems: cveItems, + } + if err := writeFile(filepath.Join(nvdDir, fmt.Sprintf("%d.json", fileNo)), feed); err != nil { + return fmt.Errorf("writing to file: %w", err) + } + + fileNo++ + totalVulns += len(cveItems) + log.Infof("Loaded %d NVD vulnerabilities", totalVulns) + // Reduce, reuse, and recycle. + cveItems = cveItems[:0] + } } - attr, err := wfn.UnbindFmtString(cpeMatch.Cpe23Uri) + + // Rudimentary rate-limiting. + // NVD limits users without an API key to roughly one call every 6 seconds. + // As of writing there are ~216,000 vulnerabilities, so this whole process should take ~11 minutes. + time.Sleep(6 * time.Second) + + startIdx += apiResp.ResultsPerPage + apiResp, err = query(fmt.Sprintf(urlFmt, startIdx)) if err != nil { - log.Errorf("error parsing %+v", item) - continue - } - if attr.Product == wfn.Any { - log.Warnf("Filtering out CPE: %+v", attr) - continue + return err } - cpeMatches = append(cpeMatches, cpeMatch) } - item.CPEMatch = cpeMatches - for _, child := range item.Children { - removeInvalidCPEs(child) + + // Write the remaining vulnerabilities. + if len(cveItems) != 0 { + enrichCVEItems(&cveItems, enrichments) + + feed := &jsonschema.NVDCVEFeedJSON10{ + CVEItems: cveItems, + } + if err := writeFile(filepath.Join(nvdDir, fmt.Sprintf("%d.json", fileNo)), feed); err != nil { + return fmt.Errorf("writing to file: %w", err) + } + + totalVulns += len(cveItems) + log.Infof("Loaded %d NVD vulnerabilities", totalVulns) } + + return nil } -func downloadFeedForYear(enrichmentMap map[string]*FileFormatWrapper, outputDir string, year int) error { - url := jsonFeedURLForYear(year) +func query(url string) (*apischema.CVEAPIJSON20, error) { + log.Debugf("Querying %s", url) resp, err := client.Get(url) if err != nil { - return errors.Wrapf(err, "failed to download feed for year %d", year) + return nil, fmt.Errorf("fetching initial NVD API results: %w", err) } defer utils.IgnoreError(resp.Body.Close) - // Un-gzip it. - gr, err := gzip.NewReader(resp.Body) - if err != nil { - return errors.Wrapf(err, "couldn't read resp body for year %d", year) + + log.Debugf("Queried %s with status code %d", url, resp.StatusCode) + if resp.StatusCode != 200 { + return nil, fmt.Errorf("unexpected status code when querying %s: %d", url, resp.StatusCode) } - // Strip out tabs and newlines for size savings - dump, err := LoadJSONFileFromReader(gr) - if err != nil { - return errors.Wrapf(err, "could not decode json for year %d", year) + apiResp := new(apischema.CVEAPIJSON20) + if err := json.NewDecoder(resp.Body).Decode(apiResp); err != nil { + return nil, fmt.Errorf("decoding API response: %w", err) } - cveItems := dump.CVEItems[:0] - for _, item := range dump.CVEItems { + return apiResp, nil +} + +func enrichCVEItems(cveItems *[]*jsonschema.NVDCVEFeedJSON10DefCVEItem, enrichments map[string]*FileFormatWrapper) { + if cveItems == nil { + return + } + + cves := (*cveItems)[:0] + for _, item := range *cveItems { if _, ok := manuallyEnrichedVulns[item.CVE.CVEDataMeta.ID]; ok { log.Warnf("Skipping vuln %s because it is being manually enriched", item.CVE.CVEDataMeta.ID) continue } + for _, node := range item.Configurations.Nodes { removeInvalidCPEs(node) } - if enrichedEntry, ok := enrichmentMap[item.CVE.CVEDataMeta.ID]; ok { + + if enrichedEntry, ok := enrichments[item.CVE.CVEDataMeta.ID]; ok { // Add the CPE matches instead of removing for backwards compatibility purposes - item.Configurations.Nodes = append(item.Configurations.Nodes, &schema.NVDCVEFeedJSON10DefNode{ + item.Configurations.Nodes = append(item.Configurations.Nodes, &jsonschema.NVDCVEFeedJSON10DefNode{ CPEMatch: enrichedEntry.AffectedPackages, Operator: "OR", }) item.LastModifiedDate = enrichedEntry.LastUpdated } - cveItems = append(cveItems, item) + cves = append(cves, item) } + for _, item := range manuallyEnrichedVulns { - cveItems = append(cveItems, item) + cves = append(cves, item) } - dump.CVEItems = cveItems - outF, err := os.Create(filepath.Join(outputDir, fmt.Sprintf("%d.json", year))) + *cveItems = cves +} + +func removeInvalidCPEs(item *jsonschema.NVDCVEFeedJSON10DefNode) { + cpeMatches := item.CPEMatch[:0] + for _, cpeMatch := range item.CPEMatch { + if cpeMatch.Cpe23Uri == "" { + cpeMatches = append(cpeMatches, cpeMatch) + continue + } + attr, err := wfn.UnbindFmtString(cpeMatch.Cpe23Uri) + if err != nil { + log.Errorf("error parsing %+v", item) + continue + } + if attr.Product == wfn.Any { + log.Warnf("Filtering out CPE: %+v", attr) + continue + } + cpeMatches = append(cpeMatches, cpeMatch) + } + item.CPEMatch = cpeMatches + for _, child := range item.Children { + removeInvalidCPEs(child) + } +} + +func writeFile(path string, feed *jsonschema.NVDCVEFeedJSON10) error { + outF, err := os.Create(path) if err != nil { - return errors.Wrap(err, "failed to create file") + return fmt.Errorf("failed to create file %s: %w", outF.Name(), err) } defer utils.IgnoreError(outF.Close) - if err := json.NewEncoder(outF).Encode(&dump); err != nil { - return errors.Wrapf(err, "could not encode json map for year %d", year) + if err := json.NewEncoder(outF).Encode(feed); err != nil { + return fmt.Errorf("could not encode JSON for %s: %w", outF.Name(), err) } - return nil -} -func jsonFeedURLForYear(year int) string { - return fmt.Sprintf("https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%d.json.gz", year) + return nil } diff --git a/pkg/vulnloader/nvdloaderv2/loader.go b/pkg/vulnloader/nvdloaderv2/loader.go deleted file mode 100644 index 064b7e508..000000000 --- a/pkg/vulnloader/nvdloaderv2/loader.go +++ /dev/null @@ -1,65 +0,0 @@ -package nvdloaderv2 - -import ( - "encoding/json" - "io" - "net/http" - "time" - - "github.com/facebookincubator/nvdtools/cveapi/nvd/schema" - "github.com/pkg/errors" - "github.com/stackrox/rox/pkg/httputil/proxy" - "github.com/stackrox/scanner/pkg/vulndump" - "github.com/stackrox/scanner/pkg/vulnloader" -) - -const ( - baseURL = `https://services.nvd.nist.gov/rest/json/cves/2.0` -) - -var ( - client = http.Client{ - Timeout: 2 * time.Minute, - Transport: proxy.RoundTripper(), - } -) - -func init() { - vulnloader.RegisterLoader(vulndump.NVDDirName, &loader{}) -} - -var _ vulnloader.Loader = (*loader)(nil) - -type loader struct {} - -// DownloadFeedsToPath downloads the NVD CVEs to the given path. -// If this function is successful, it will fill the directory with -// one json file for each year of NVD data. -func (l *loader) DownloadFeedsToPath(s string) error { - // Just do a basic query at the start. - // This will help us determine the default resultsPerPage, - // which is recommended to use. It will also use startIndex=0 - // automatically. - resp, err := client.Get(baseURL) - if err != nil { - return errors.Wrap(err, "fetching initial NVD API results") - } - apiResponse, err := parse(resp.Body) - if err != nil { - return errors.Wrapf(err, "parsing body for API request to %q", baseURL) - } - -} - -func parse(body io.ReadCloser) (*schema.CVEAPIJSON20, error) { - defer func() { - _ = body.Close() - }() - - apiResponse := new(schema.CVEAPIJSON20) - if err := json.NewDecoder(body).Decode(apiResponse); err != nil { - return nil, errors.Wrap(err, "decoding API response") - } - - return apiResponse, nil -}