Skip to content

Commit 6ed00e5

Browse files
RTannRTann
committed
all: account for language package overwrites
Signed-off-by: RTann <[email protected]>
1 parent 7250a05 commit 6ed00e5

12 files changed

+254
-262
lines changed

‎gobin/coalescer.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
1616
Packages: map[string]*claircore.Package{},
1717
Repositories: map[string]*claircore.Repository{},
1818
}
19-
for _, l := range ls {
19+
// Similar to ir.Packages, except instead of mapping
20+
// id -> package, it maps packageDB -> package.
21+
// For langauge packages, it is possible the
22+
// packageDB is overwritten.
23+
packages := make(map[string]*claircore.Package)
24+
for i := len(ls) - 1; i >= 0; i-- {
25+
l := ls[i]
2026
var rid string
2127
for _, r := range l.Repos {
2228
// Magic strings copied out of the osv package.
@@ -31,6 +37,19 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
3137
if !strings.HasPrefix(pkg.PackageDB, "go:") {
3238
continue
3339
}
40+
if childPkg, exists := packages[pkg.PackageDB]; exists {
41+
// If the package was renamed or has a different version in a high layer,
42+
// then we consider this a different package and ignore the
43+
// original in the lower layer.
44+
if pkg.Name != childPkg.Name || pkg.Version != childPkg.Version {
45+
continue
46+
}
47+
// The name and version is the same, so delete the entry related to the higher
48+
// layer, as this package was likely introduced in the lower layer.
49+
delete(ir.Packages, childPkg.ID)
50+
delete(ir.Environments, childPkg.ID)
51+
}
52+
packages[pkg.PackageDB] = pkg
3453
ir.Packages[pkg.ID] = pkg
3554
ir.Environments[pkg.ID] = []*claircore.Environment{
3655
{

‎java/coalescer.go

-42
This file was deleted.

‎java/ecosystem.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ import (
44
"context"
55

66
"github.com/quay/claircore/indexer"
7+
"github.com/quay/claircore/language"
78
)
89

910
// NewEcosystem provides the set of scanners for the java ecosystem.
10-
func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
11+
func NewEcosystem(_ context.Context) *indexer.Ecosystem {
1112
return &indexer.Ecosystem{
1213
PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
1314
return []indexer.PackageScanner{&Scanner{}}, nil
@@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
1617
RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) {
1718
return nil, nil
1819
},
19-
Coalescer: func(_ context.Context) (indexer.Coalescer, error) {
20-
return (*coalescer)(nil), nil
21-
},
20+
Coalescer: language.NewCoalescer,
2221
}
2322
}

‎language/coalescer.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
// Package language implements structs and functions common between
2+
// programming language indexing implementation.
3+
package language
4+
5+
import (
6+
"context"
7+
8+
"github.com/quay/claircore"
9+
"github.com/quay/claircore/indexer"
10+
)
11+
12+
var _ indexer.Coalescer = (*coalescer)(nil)
13+
14+
type coalescer struct{}
15+
16+
// NewCoalescer returns a new common programming language coalescer.
17+
func NewCoalescer(_ context.Context) (indexer.Coalescer, error) {
18+
return &coalescer{}, nil
19+
}
20+
21+
// Coalesce implements [indexer.Coalescer].
22+
func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*claircore.IndexReport, error) {
23+
ir := &claircore.IndexReport{
24+
Environments: map[string][]*claircore.Environment{},
25+
Packages: map[string]*claircore.Package{},
26+
Repositories: map[string]*claircore.Repository{},
27+
}
28+
// Similar to ir.Packages, except instead of mapping
29+
// id -> package, it maps packageDB -> package.
30+
// For programming language packages, it is possible the
31+
// packageDB is overwritten.
32+
packages := make(map[string]*claircore.Package)
33+
for i := len(ls) - 1; i >= 0; i-- {
34+
l := ls[i]
35+
// If we didn't find at least one repo in this layer
36+
// no point in searching for packages.
37+
if len(l.Repos) == 0 {
38+
continue
39+
}
40+
rs := make([]string, len(l.Repos))
41+
for i, r := range l.Repos {
42+
rs[i] = r.ID
43+
ir.Repositories[r.ID] = r
44+
}
45+
for _, pkg := range l.Pkgs {
46+
if seen, exists := packages[pkg.PackageDB]; exists {
47+
// If the package was renamed or has a different version in a high layer,
48+
// then we consider this a different package and ignore the
49+
// original in the lower layer.
50+
if pkg.Name != seen.Name || pkg.Version != seen.Version {
51+
continue
52+
}
53+
// The name and version is the same, so delete the entry related to the higher
54+
// layer, as this package was likely introduced in the lower layer.
55+
delete(ir.Packages, seen.ID)
56+
delete(ir.Environments, seen.ID)
57+
}
58+
packages[pkg.PackageDB] = pkg
59+
ir.Packages[pkg.ID] = pkg
60+
ir.Environments[pkg.ID] = []*claircore.Environment{
61+
{
62+
PackageDB: pkg.PackageDB,
63+
IntroducedIn: l.Hash,
64+
RepositoryIDs: rs,
65+
},
66+
}
67+
}
68+
}
69+
return ir, nil
70+
}

‎language/coalescer_test.go

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package language
2+
3+
import (
4+
"context"
5+
"strconv"
6+
"testing"
7+
8+
"github.com/quay/zlog"
9+
10+
"github.com/quay/claircore"
11+
"github.com/quay/claircore/indexer"
12+
"github.com/quay/claircore/test"
13+
)
14+
15+
func TestCoalescer(t *testing.T) {
16+
t.Parallel()
17+
ctx := zlog.Test(context.Background(), t)
18+
coalescer := &coalescer{}
19+
pkgs := test.GenUniquePackages(6)
20+
repo := []*claircore.Repository{{
21+
Name: "npm",
22+
URI: "https://www.npmjs.com/",
23+
}}
24+
layerArtifacts := []*indexer.LayerArtifacts{
25+
{
26+
Hash: test.RandomSHA256Digest(t),
27+
Pkgs: pkgs[:1],
28+
},
29+
{
30+
Hash: test.RandomSHA256Digest(t),
31+
Pkgs: pkgs[:2],
32+
},
33+
{
34+
Hash: test.RandomSHA256Digest(t),
35+
Pkgs: pkgs[:3],
36+
Repos: repo,
37+
},
38+
{
39+
Hash: test.RandomSHA256Digest(t),
40+
Pkgs: pkgs[:4],
41+
},
42+
{
43+
Hash: test.RandomSHA256Digest(t),
44+
Pkgs: pkgs[:5],
45+
Repos: repo,
46+
},
47+
{
48+
Hash: test.RandomSHA256Digest(t),
49+
Pkgs: pkgs,
50+
},
51+
}
52+
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
53+
if err != nil {
54+
t.Fatalf("received error from coalesce method: %v", err)
55+
}
56+
// Expect 0-5 to have gotten associated with the repository.
57+
for i := range pkgs {
58+
es, ok := ir.Environments[strconv.Itoa(i)]
59+
if !ok && i == 5 {
60+
// Left out the last package.
61+
continue
62+
}
63+
e := es[0]
64+
if len(e.RepositoryIDs) == 0 {
65+
t.Error("expected some repositories")
66+
}
67+
for _, id := range e.RepositoryIDs {
68+
r := ir.Repositories[id]
69+
if got, want := r.Name, "npm"; got != want {
70+
t.Errorf("got: %q, want: %q", got, want)
71+
}
72+
}
73+
}
74+
}
75+
76+
func TestCoalescer_package_overwrite(t *testing.T) {
77+
t.Parallel()
78+
ctx := zlog.Test(context.Background(), t)
79+
coalescer := &coalescer{}
80+
repo := []*claircore.Repository{{
81+
Name: "npm",
82+
URI: "https://www.npmjs.com/",
83+
}}
84+
hashes := []claircore.Digest{
85+
test.RandomSHA256Digest(t),
86+
test.RandomSHA256Digest(t),
87+
test.RandomSHA256Digest(t),
88+
test.RandomSHA256Digest(t),
89+
}
90+
layerArtifacts := []*indexer.LayerArtifacts{
91+
{
92+
Hash: hashes[0],
93+
Pkgs: []*claircore.Package{
94+
{
95+
ID: "0",
96+
Name: "semver",
97+
Version: "7.3.8",
98+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
99+
},
100+
},
101+
Repos: repo,
102+
},
103+
{
104+
Hash: hashes[1],
105+
},
106+
{
107+
Hash: hashes[2],
108+
Pkgs: []*claircore.Package{
109+
{
110+
ID: "1",
111+
Name: "semver",
112+
Version: "7.5.2",
113+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
114+
},
115+
},
116+
Repos: repo,
117+
},
118+
{
119+
Hash: hashes[3],
120+
Pkgs: []*claircore.Package{
121+
{
122+
ID: "2",
123+
Name: "semver",
124+
Version: "7.5.2",
125+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
126+
},
127+
},
128+
Repos: repo,
129+
},
130+
}
131+
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
132+
if err != nil {
133+
t.Fatalf("received error from coalesce method: %v", err)
134+
}
135+
if len(ir.Packages) != 1 {
136+
t.Fatalf("unexpected number of packages: %d != %d", len(ir.Packages), 1)
137+
}
138+
pkg, exists := ir.Packages["1"]
139+
if !exists {
140+
t.Fatal("expected package does not exist")
141+
}
142+
if pkg.Version != "7.5.2" {
143+
t.Fatalf("unexpected version: %s != %s", pkg.Version, "7.5.2")
144+
}
145+
envs, exists := ir.Environments["1"]
146+
if !exists {
147+
t.Fatal("expected environments do not exist")
148+
}
149+
if len(envs) != 1 {
150+
t.Fatalf("unexpected number of envionments: %d != %d", len(envs), 1)
151+
}
152+
if envs[0].IntroducedIn.String() != hashes[2].String() {
153+
t.Fatalf("unexpected introducedIn: %s != %s", envs[0].IntroducedIn.String(), hashes[2].String())
154+
}
155+
}

‎nodejs/coalescer.go

-48
This file was deleted.

0 commit comments

Comments
 (0)