rpm: add function to determine if paths are RPM

Allow language indexers to check if the filepaths they are interested in
have been installed via RPM.

Signed-off-by: crozzy <[email protected]>

Author: crozzy

file.go

@@ -5,6 +5,7 @@ type FileKind string
 
 const (
 	FileKindWhiteout = FileKind("whiteout")
+	FileKindRPM      = FileKind("rpm")
 )
 
 // File represents interesting files that are found in the layer.

gobin/gobin.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/indexer"
+	"github.com/quay/claircore/rpm"
 )
 
 // Detector detects go binaries and reports the packages used to build them.
@@ -86,7 +87,8 @@ func (Detector) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Pack
 	// Only create a single spool file per call, re-use for every binary.
 	var spool spoolfile
 	walk := func(p string, d fs.DirEntry, err error) error {
-		ctx := zlog.ContextWithValues(ctx, "path", d.Name())
+		ctx := zlog.ContextWithValues(ctx, "filename", d.Name())
+
 		switch {
 		case err != nil:
 			return err
@@ -107,6 +109,18 @@ func (Detector) Scan(ctx context.Context, l *claircore.Layer) ([]*claircore.Pack
 			// Not executable
 			return nil
 		}
+
+		isRPM, err := rpm.FileInstalledByRPM(ctx, l, p)
+		if err != nil {
+			return err
+		}
+		if isRPM {
+			zlog.Debug(ctx).
+				Str("path", p).
+				Msg("file path determined to be of RPM origin")
+			return nil
+		}
+
 		f, err := sys.Open(p)
 		if err != nil {
 			// TODO(crozzy): Remove log line once controller is in a

python/packagescanner.go

@@ -19,6 +19,7 @@ import (
 	"github.com/quay/claircore"
 	"github.com/quay/claircore/indexer"
 	"github.com/quay/claircore/pkg/pep440"
+	"github.com/quay/claircore/rpm"
 )
 
 var (
@@ -79,6 +80,16 @@ func (ps *Scanner) Scan(ctx context.Context, layer *claircore.Layer) ([]*clairco
 	}
 	var ret []*claircore.Package
 	for _, n := range ms {
+		isRPM, err := rpm.FileInstalledByRPM(ctx, layer, n)
+		if err != nil {
+			return nil, err
+		}
+		if isRPM {
+			zlog.Debug(ctx).
+				Str("path", n).
+				Msg("file path determined to be of RPM origin")
+			continue
+		}
 		b, err := fs.ReadFile(sys, n)
 		if err != nil {
 			return nil, fmt.Errorf("python: unable to read file: %w", err)
@@ -143,14 +154,14 @@ func findDeliciousEgg(ctx context.Context, sys fs.FS) (out []string, err error)
 	// Is this layer an rpm layer?
 	//
 	// If so, files in the disto-managed directory can be skipped.
-	var rpm bool
+	var isRPM bool
 	for _, p := range []string{
 		"var/lib/rpm/Packages",
 		"var/lib/rpm/rpmdb.sqlite",
 		"var/lib/rpm/Packages.db",
 	} {
 		if fi, err := fs.Stat(sys, p); err == nil && fi.Mode().IsRegular() {
-			rpm = true
+			isRPM = true
 			break
 		}
 	}
@@ -172,12 +183,12 @@ func findDeliciousEgg(ctx context.Context, sys fs.FS) (out []string, err error)
 		switch {
 		case err != nil:
 			return err
-		case (rpm || dpkg) && d.Type().IsDir():
+		case (isRPM || dpkg) && d.Type().IsDir():
 			// Skip one level up from the "packages" directory so the walk also
 			// skips the standard library.
 			var pat string
 			switch {
-			case rpm:
+			case isRPM:
 				pat = `usr/lib*/python[23].*`
 				ev = ev.Bool("rpm_dir", true)
 			case dpkg:

rpm/files.go

@@ -0,0 +1,102 @@
+package rpm
+
+import (
+	"context"
+	"fmt"
+	"io/fs"
+	"sync"
+
+	"github.com/quay/claircore"
+	"github.com/quay/zlog"
+)
+
+// filesCache is used for concurrent access to the map containing layer.Hash -> map RPM files.
+// The value is a map to allow for quick member checking.
+type filesCache struct {
+	c  map[string]map[string]struct{}
+	mu *sync.Mutex
+}
+
+var fc *filesCache
+
+func init() {
+	fc = &filesCache{
+		c:  map[string]map[string]struct{}{},
+		mu: &sync.Mutex{},
+	}
+}
+
+// gc deletes the layer's entry from the map if the ctx is done, this ties the lifecycle of
+// the cached information to the request lifecycle to avoid excessive memory consumption.
+func (fc *filesCache) gc(ctx context.Context, key string) {
+	<-ctx.Done()
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+	delete(fc.c, key)
+}
+
+// getFiles looks up RPM files that exist in the RPM database using the filesFromDB
+// function and memorizes the result to avoid repeated work for the same claircore.Layer.
+func (fc *filesCache) getFiles(ctx context.Context, layer *claircore.Layer) (map[string]struct{}, error) {
+	if fc == nil {
+		panic("programmer error: filesCache nil")
+	}
+	fc.mu.Lock()
+	defer fc.mu.Unlock()
+	if files, ok := fc.c[layer.Hash.String()]; ok {
+		return files, nil
+	}
+
+	sys, err := layer.FS()
+	if err != nil {
+		return nil, fmt.Errorf("rpm: unable to open layer: %w", err)
+	}
+
+	files := map[string]struct{}{}
+	defer func() {
+		// Defer setting the cache so any early-outs don't have to worry.
+		fc.c[layer.Hash.String()] = files
+	}()
+	found := make([]foundDB, 0)
+	if err := fs.WalkDir(sys, ".", findDBs(ctx, &found, sys)); err != nil {
+		return nil, fmt.Errorf("rpm: error walking fs: %w", err)
+	}
+	if len(found) == 0 {
+		return nil, nil
+	}
+
+	done := map[string]struct{}{}
+	zlog.Debug(ctx).Int("count", len(found)).Msg("found possible databases")
+	for _, db := range found {
+		ctx := zlog.ContextWithValues(ctx, "db", db.String())
+		zlog.Debug(ctx).Msg("examining database")
+		if _, ok := done[db.Path]; ok {
+			zlog.Debug(ctx).Msg("already seen, skipping")
+			continue
+		}
+		done[db.Path] = struct{}{}
+		fs, err := getDBObjects(ctx, sys, db, filesFromDB)
+		if err != nil {
+			return nil, fmt.Errorf("rpm: error getting native DBs: %w", err)
+		}
+		for _, f := range fs {
+			files[f.Path] = struct{}{}
+		}
+	}
+	go func() {
+		fc.gc(ctx, layer.Hash.String())
+	}()
+
+	return files, nil
+}
+
+// FileInstalledByRPM takes a claircore.Layer and filepath string and returns a boolean
+// signifying whether that file came from an RPM package.
+func FileInstalledByRPM(ctx context.Context, layer *claircore.Layer, filepath string) (bool, error) {
+	files, err := fc.getFiles(ctx, layer)
+	if err != nil {
+		return false, err
+	}
+	_, exists := files[filepath]
+	return exists, nil
+}

rpm/files_test.go

@@ -0,0 +1,77 @@
+package rpm
+
+import (
+	"context"
+	"testing"
+
+	"github.com/quay/claircore"
+	"github.com/quay/claircore/test"
+	"github.com/quay/zlog"
+)
+
+var rpmFilesTestcases = []struct {
+	name     string
+	isRPM    bool
+	filePath string
+	layer    test.LayerRef
+	lenFiles int
+}{
+	{
+		name:     "Found",
+		isRPM:    true,
+		filePath: "usr/lib/node_modules/npm/node_modules/safe-buffer/package.json",
+		layer: test.LayerRef{
+			Registry: "registry.access.redhat.com",
+			Name:     "ubi9/nodejs-18",
+			Digest:   `sha256:1ae06b64755052cef4c32979aded82a18f664c66fa7b50a6d2924afac2849c6e`,
+		},
+		lenFiles: 100,
+	},
+	{
+		name:     "Not found",
+		isRPM:    false,
+		filePath: "usr/lib/node_modules/npm/node_modules/safe-buffer/package.jsonx",
+		layer: test.LayerRef{
+			Registry: "registry.access.redhat.com",
+			Name:     "ubi9/nodejs-18",
+			Digest:   `sha256:1ae06b64755052cef4c32979aded82a18f664c66fa7b50a6d2924afac2849c6e`,
+		},
+		lenFiles: 100,
+	},
+}
+
+func TestIsRPMFile(t *testing.T) {
+	ctx := zlog.Test(context.Background(), t)
+	a := test.NewCachedArena(t)
+
+	for _, tt := range rpmFilesTestcases {
+		t.Run(tt.name, func(t *testing.T) {
+			a.LoadLayerFromRegistry(ctx, t, tt.layer)
+			r := a.Realizer(ctx).(*test.CachedRealizer)
+			t.Cleanup(func() {
+				if err := r.Close(); err != nil {
+					t.Error(err)
+				}
+			})
+
+			realizedLayers, err := r.RealizeDescriptions(ctx, []claircore.LayerDescription{
+				{
+					Digest:    tt.layer.Digest,
+					URI:       "http://example.com",
+					MediaType: test.MediaType,
+					Headers:   make(map[string][]string),
+				},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			isRPM, err := FileInstalledByRPM(ctx, &realizedLayers[0], tt.filePath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if tt.isRPM != isRPM {
+				t.Errorf("expected isRPM: %t, got isRPM: %t", tt.isRPM, isRPM)
+			}
+		})
+	}
+}