rhel: WIP: repoids

Signed-off-by: Hank Donnay <hdonnay@redhat.com>

rhel/coalescer.go

@@ -2,6 +2,7 @@ package rhel
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/indexer"
@@ -116,10 +117,15 @@ func (*Coalescer) Coalesce(ctx context.Context, artifacts []*indexer.LayerArtifa
 					PackageDB:      pkg.PackageDB,
 					IntroducedIn:   layerArtifacts.Hash,
 					DistributionID: distID,
-					RepositoryIDs:  make([]string, len(layerArtifacts.Repos)),
 				}
-				for i := range layerArtifacts.Repos {
-					environment.RepositoryIDs[i] = layerArtifacts.Repos[i].ID
+				v, _ := url.ParseQuery(pkg.RepositoryHint)
+				if id := v.Get("repoid"); id != "" {
+					environment.RepositoryIDs = v["repoid"]
+				} else {
+					environment.RepositoryIDs = make([]string, len(layerArtifacts.Repos))
+					for i := range layerArtifacts.Repos {
+						environment.RepositoryIDs[i] = layerArtifacts.Repos[i].ID
+					}
 				}
 				db.packages[pkg.ID] = pkg
 				db.environments[pkg.ID] = environment

rhel/ecosystem.go

@@ -4,14 +4,13 @@ import (
 	"context"
 
 	"github.com/quay/claircore/indexer"
-	"github.com/quay/claircore/rpm"
 )
 
 // NewEcosystem provides the set of scanners and coalescer for the rhel ecosystem.
 func NewEcosystem(_ context.Context) *indexer.Ecosystem {
 	return &indexer.Ecosystem{
 		PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
-			return []indexer.PackageScanner{new(rpm.Scanner)}, nil
+			return []indexer.PackageScanner{PackageScanner{}}, nil
 		},
 		DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) {
 			return []indexer.DistributionScanner{new(DistributionScanner)}, nil

rhel/matcher_test.go

@@ -272,13 +272,13 @@ func TestIsCPEStringSubsetMatch(t *testing.T) {
 		match              bool
 	}{
 		{
-			name:      "simple_case",
+			name:      "Simple",
 			recordCPE: cpe.MustUnbind("cpe:/a:redhat:openshift:4.13::el8"),
 			vulnCPE:   cpe.MustUnbind("cpe:/a:redhat:openshift:4"),
 			match:     true,
 		},
 		{
-			name:      "wrong_minor",
+			name:      "WrongMinor",
 			recordCPE: cpe.MustUnbind("cpe:/a:redhat:openshift:4.13::el8"),
 			vulnCPE:   cpe.MustUnbind("cpe:/a:redhat:openshift:4.1::el8"),
 			match:     false,

rhel/packagescanner.go

@@ -0,0 +1,92 @@
+package rhel
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"runtime/trace"
+	"slices"
+
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/internal/dnf"
+	"github.com/quay/claircore/internal/rpm"
+	"github.com/quay/claircore/internal/wart"
+)
+
+var _ indexer.PackageScanner = PackageScanner{}
+
+type PackageScanner struct{}
+
+// Kind implements [indexer.PackageScanner].
+func (p PackageScanner) Kind() string { return "package" }
+
+// Name implements [indexer.PackageScanner].
+func (p PackageScanner) Name() string { return "rhel-package-scanner" }
+
+// Version implements [indexer.PackageScanner].
+func (p PackageScanner) Version() string { return "1" }
+
+// Scan implements [indexer.PackageScanner].
+func (p PackageScanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*claircore.Package, error) {
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	defer trace.StartRegion(ctx, "PackageScanner.Scan").End()
+	trace.Log(ctx, "layer", layer.Hash.String())
+	ctx = zlog.ContextWithValues(ctx,
+		"component", "rhel/PackageScanner.Scan",
+		"version", p.Version(),
+		"layer", layer.Hash.String())
+	zlog.Debug(ctx).Msg("start")
+	defer zlog.Debug(ctx).Msg("done")
+
+	sys, err := layer.FS()
+	if err != nil {
+		return nil, fmt.Errorf("rhel: unable to open layer: %w", err)
+	}
+
+	found, err := rpm.FindDBs(ctx, sys)
+	if err != nil {
+		return nil, fmt.Errorf("rhel: error examining fs: %w", err)
+	}
+	if len(found) == 0 {
+		return nil, nil
+	}
+
+	zlog.Debug(ctx).Int("count", len(found)).Msg("found possible databases")
+
+	a, err := dnf.NewAnnotator(ctx, sys)
+	if err != nil {
+		return nil, fmt.Errorf("rhel: error examining fs: %w", err)
+	}
+	defer func() {
+		if err := a.Close(); err != nil {
+			zlog.Warn(ctx).Err(err).Msg("error closing dnf Annotator")
+		}
+	}()
+	if a == dnf.Identity {
+		zlog.Debug(ctx).Msg("no dnf information found")
+	}
+
+	var final error
+	seq := func(yield func(*claircore.Package) bool) {
+		for _, db := range found {
+			ctx := zlog.ContextWithValues(ctx, "db", db.String())
+			zlog.Debug(ctx).Msg("examining database")
+
+			seq, checkPkgs := db.Packages(ctx)
+			seq, checkDNF := a.Wrap(ctx, seq)
+			wart.AsPointer(seq)(yield)
+
+			if err := errors.Join(checkDNF(), checkPkgs()); err != nil {
+				final = fmt.Errorf("rhel: error reading native db: %w", err)
+				return
+			}
+		}
+	}
+
+	return slices.Collect(seq), final
+}

rhel/packagescanner_test.go

@@ -0,0 +1,119 @@
+package rhel
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/quay/zlog"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/test"
+	"github.com/quay/claircore/test/rpmtest"
+)
+
+func TestPackageScanner(t *testing.T) {
+	t.Parallel()
+	ctx := zlog.Test(context.Background(), t)
+
+	a := test.NewCachedArena(t)
+	defer func() {
+		if err := a.Close(ctx); err != nil {
+			t.Error(err)
+		}
+	}()
+
+	tt := []PackageTestcase{
+		{
+			Name:         "BDB",
+			ManifestFile: "bdb.rpm-manifest.json",
+			// Some forgotten CentOS layer.
+			Ref: test.LayerRef{
+				Registry: "docker.io",
+				Name:     "library/centos",
+				Digest:   `sha256:729ec3a6ada3a6d26faca9b4779a037231f1762f759ef34c08bdd61bf52cd704`,
+			},
+		},
+		{
+			Name:         "NodeJS",
+			ManifestFile: "nodejs.rpm-manifest.json",
+			// Layer from registry.access.redhat.com/ubi9/nodejs-18@sha256:1ff5080686736cbab820ec560873c59bd80659a2b2f8d8f4e379301a910e5d54
+			Ref: test.LayerRef{
+				Registry: "registry.access.redhat.com",
+				Name:     "ubi9/nodejs-18",
+				Digest:   `sha256:1ae06b64755052cef4c32979aded82a18f664c66fa7b50a6d2924afac2849c6e`,
+			},
+		},
+		{
+			Name:         "Httpd24NoContentSets",
+			ManifestFile: "httpd-24_9.5-1734525854.rpm-manifest.json",
+			// Layer from registry.access.redhat.com/ubi9/httpd-24:9.5-1734525854
+			Ref: test.LayerRef{
+				Registry: "registry.access.redhat.com",
+				Name:     "ubi9/httpd-24",
+				Digest:   `sha256:572f60f98d5ae116073fa5f8c576fc014afdcd4c68875e37c37032ad2772f653`,
+			},
+		},
+	}
+	for _, tc := range tt {
+		t.Run(tc.Name, tc.Run(ctx, a))
+	}
+}
+
+type PackageTestcase struct {
+	Name         string
+	ManifestFile string
+	Ref          test.LayerRef
+}
+
+func (tc PackageTestcase) Run(ctx context.Context, a *test.CachedArena) func(*testing.T) {
+	s := &PackageScanner{}
+	return func(t *testing.T) {
+		t.Parallel()
+		ctx := zlog.Test(ctx, t)
+		a.LoadLayerFromRegistry(ctx, t, tc.Ref)
+		wf, err := os.Open(filepath.Join("testdata/", tc.ManifestFile))
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() {
+			if err := wf.Close(); err != nil {
+				t.Error(err)
+			}
+		})
+		want := rpmtest.PackagesFromRPMManifest(t, wf)
+		r := a.Realizer(ctx).(*test.CachedRealizer)
+		t.Cleanup(func() {
+			if err := r.Close(); err != nil {
+				t.Error(err)
+			}
+		})
+		ls, err := r.RealizeDescriptions(ctx, []claircore.LayerDescription{
+			{
+				Digest:    tc.Ref.Digest,
+				URI:       "http://example.com",
+				MediaType: test.MediaType,
+				Headers:   make(map[string][]string),
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		got, err := s.Scan(ctx, &ls[0])
+		if err != nil {
+			t.Error(err)
+		}
+		t.Logf("found %d packages", len(got))
+		/*
+			for _, pkg := range got {
+				t.Logf("%s: %#q", pkg.Name, pkg.RepositoryHint)
+			}
+		*/
+		if !cmp.Equal(got, want, rpmtest.Options) {
+			t.Error(cmp.Diff(got, want, rpmtest.Options))
+		}
+	}
+}