Skip to content

Commit 3637e69

Browse files
RTannRTann
committed
all: account for language package overwrites
Signed-off-by: RTann <[email protected]>
1 parent 0831b93 commit 3637e69

11 files changed

+207
-219
lines changed

‎gobin/coalescer.go

+20-1
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,13 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
1616
Packages: map[string]*claircore.Package{},
1717
Repositories: map[string]*claircore.Repository{},
1818
}
19-
for _, l := range ls {
19+
// Similar to ir.Packages, except instead of mapping
20+
// id -> package, it maps packageDB -> package.
21+
// For langauge packages, it is possible the
22+
// packageDB is overwritten.
23+
packages := make(map[string]*claircore.Package)
24+
for i := len(ls) - 1; i >= 0; i-- {
25+
l := ls[i]
2026
var rid string
2127
for _, r := range l.Repos {
2228
// Magic strings copied out of the osv package.
@@ -31,6 +37,19 @@ func (c *coalescer) Coalesce(ctx context.Context, ls []*indexer.LayerArtifacts)
3137
if !strings.HasPrefix(pkg.PackageDB, "go:") {
3238
continue
3339
}
40+
if childPkg, exists := packages[pkg.PackageDB]; exists {
41+
// If the package was renamed or has a different version in a high layer,
42+
// then we consider this a different package and ignore the
43+
// original in the lower layer.
44+
if pkg.Name != childPkg.Name || pkg.Version != childPkg.Version {
45+
continue
46+
}
47+
// The name and version is the same, so delete the entry related to the higher
48+
// layer, as this package was likely introduced in the lower layer.
49+
delete(ir.Packages, childPkg.ID)
50+
delete(ir.Environments, childPkg.ID)
51+
}
52+
packages[pkg.PackageDB] = pkg
3453
ir.Packages[pkg.ID] = pkg
3554
ir.Environments[pkg.ID] = []*claircore.Environment{
3655
{

‎java/coalescer.go

-42
This file was deleted.

‎java/ecosystem.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ import (
44
"context"
55

66
"github.com/quay/claircore/indexer"
7+
"github.com/quay/claircore/language"
78
)
89

910
// NewEcosystem provides the set of scanners for the java ecosystem.
10-
func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
11+
func NewEcosystem(_ context.Context) *indexer.Ecosystem {
1112
return &indexer.Ecosystem{
1213
PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) {
1314
return []indexer.PackageScanner{&Scanner{}}, nil
@@ -16,8 +17,6 @@ func NewEcosystem(ctx context.Context) *indexer.Ecosystem {
1617
RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) {
1718
return nil, nil
1819
},
19-
Coalescer: func(_ context.Context) (indexer.Coalescer, error) {
20-
return (*coalescer)(nil), nil
21-
},
20+
Coalescer: language.NewCoalescer,
2221
}
2322
}

‎nodejs/coalescer.go ‎language/coalescer.go

+23-5
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
package nodejs
1+
package language
22

33
import (
44
"context"
5-
"github.com/quay/claircore"
65

6+
"github.com/quay/claircore"
77
"github.com/quay/claircore/indexer"
88
)
99

@@ -21,9 +21,14 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*
2121
Packages: map[string]*claircore.Package{},
2222
Repositories: map[string]*claircore.Repository{},
2323
}
24-
25-
for _, l := range ls {
26-
// If we didn't find at least one npm repo in this layer
24+
// Similar to ir.Packages, except instead of mapping
25+
// id -> package, it maps packageDB -> package.
26+
// For langauge packages, it is possible the
27+
// packageDB is overwritten.
28+
packages := make(map[string]*claircore.Package)
29+
for i := len(ls) - 1; i >= 0; i-- {
30+
l := ls[i]
31+
// If we didn't find at least one repo in this layer
2732
// no point in searching for packages.
2833
if len(l.Repos) == 0 {
2934
continue
@@ -34,6 +39,19 @@ func (c *coalescer) Coalesce(_ context.Context, ls []*indexer.LayerArtifacts) (*
3439
ir.Repositories[r.ID] = r
3540
}
3641
for _, pkg := range l.Pkgs {
42+
if childPkg, exists := packages[pkg.PackageDB]; exists {
43+
// If the package was renamed or has a different version in a high layer,
44+
// then we consider this a different package and ignore the
45+
// original in the lower layer.
46+
if pkg.Name != childPkg.Name || pkg.Version != childPkg.Version {
47+
continue
48+
}
49+
// The name and version is the same, so delete the entry related to the higher
50+
// layer, as this package was likely introduced in the lower layer.
51+
delete(ir.Packages, childPkg.ID)
52+
delete(ir.Environments, childPkg.ID)
53+
}
54+
packages[pkg.PackageDB] = pkg
3755
ir.Packages[pkg.ID] = pkg
3856
ir.Environments[pkg.ID] = []*claircore.Environment{
3957
{

‎language/coalescer_test.go

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
package language
2+
3+
import (
4+
"context"
5+
"strconv"
6+
"testing"
7+
8+
"github.com/quay/zlog"
9+
10+
"github.com/quay/claircore"
11+
"github.com/quay/claircore/indexer"
12+
"github.com/quay/claircore/test"
13+
)
14+
15+
func TestCoalescer(t *testing.T) {
16+
t.Parallel()
17+
ctx := zlog.Test(context.Background(), t)
18+
coalescer := &coalescer{}
19+
pkgs := test.GenUniquePackages(6)
20+
repo := []*claircore.Repository{{
21+
Name: "npm",
22+
URI: "https://www.npmjs.com/",
23+
}}
24+
layerArtifacts := []*indexer.LayerArtifacts{
25+
{
26+
Hash: test.RandomSHA256Digest(t),
27+
Pkgs: pkgs[:1],
28+
},
29+
{
30+
Hash: test.RandomSHA256Digest(t),
31+
Pkgs: pkgs[:2],
32+
},
33+
{
34+
Hash: test.RandomSHA256Digest(t),
35+
Pkgs: pkgs[:3],
36+
Repos: repo,
37+
},
38+
{
39+
Hash: test.RandomSHA256Digest(t),
40+
Pkgs: pkgs[:4],
41+
},
42+
{
43+
Hash: test.RandomSHA256Digest(t),
44+
Pkgs: pkgs[:5],
45+
Repos: repo,
46+
},
47+
{
48+
Hash: test.RandomSHA256Digest(t),
49+
Pkgs: pkgs,
50+
},
51+
}
52+
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
53+
if err != nil {
54+
t.Fatalf("received error from coalesce method: %v", err)
55+
}
56+
// Expect 0-5 to have gotten associated with the repository.
57+
for i := range pkgs {
58+
es, ok := ir.Environments[strconv.Itoa(i)]
59+
if !ok && i == 5 {
60+
// Left out the last package.
61+
continue
62+
}
63+
e := es[0]
64+
if len(e.RepositoryIDs) == 0 {
65+
t.Error("expected some repositories")
66+
}
67+
for _, id := range e.RepositoryIDs {
68+
r := ir.Repositories[id]
69+
if got, want := r.Name, "npm"; got != want {
70+
t.Errorf("got: %q, want: %q", got, want)
71+
}
72+
}
73+
}
74+
}
75+
76+
func TestCoalescer_package_overwrite(t *testing.T) {
77+
t.Parallel()
78+
ctx := zlog.Test(context.Background(), t)
79+
coalescer := &coalescer{}
80+
repo := []*claircore.Repository{{
81+
Name: "npm",
82+
URI: "https://www.npmjs.com/",
83+
}}
84+
hashes := []claircore.Digest{
85+
test.RandomSHA256Digest(t),
86+
test.RandomSHA256Digest(t),
87+
test.RandomSHA256Digest(t),
88+
test.RandomSHA256Digest(t),
89+
}
90+
layerArtifacts := []*indexer.LayerArtifacts{
91+
{
92+
Hash: hashes[0],
93+
Pkgs: []*claircore.Package{
94+
{
95+
ID: "0",
96+
Name: "semver",
97+
Version: "7.3.8",
98+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
99+
},
100+
},
101+
Repos: repo,
102+
},
103+
{
104+
Hash: hashes[1],
105+
},
106+
{
107+
Hash: hashes[2],
108+
Pkgs: []*claircore.Package{
109+
{
110+
ID: "1",
111+
Name: "semver",
112+
Version: "7.5.2",
113+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
114+
},
115+
},
116+
Repos: repo,
117+
},
118+
{
119+
Hash: hashes[3],
120+
Pkgs: []*claircore.Package{
121+
{
122+
ID: "2",
123+
Name: "semver",
124+
Version: "7.5.2",
125+
PackageDB: "nodejs:usr/local/lib/node_modules/npm/node_modules/semver/package.json",
126+
},
127+
},
128+
Repos: repo,
129+
},
130+
}
131+
ir, err := coalescer.Coalesce(ctx, layerArtifacts)
132+
if err != nil {
133+
t.Fatalf("received error from coalesce method: %v", err)
134+
}
135+
if len(ir.Packages) != 1 {
136+
t.Fatalf("unexpected number of packages: %d != %d", len(ir.Packages), 1)
137+
}
138+
pkg, exists := ir.Packages["1"]
139+
if !exists {
140+
t.Fatal("expected package does not exist")
141+
}
142+
if pkg.Version != "7.5.2" {
143+
t.Fatalf("unexpected version: %s != %s", pkg.Version, "7.5.2")
144+
}
145+
envs, exists := ir.Environments["1"]
146+
if !exists {
147+
t.Fatal("expected environments do not exist")
148+
}
149+
if len(envs) != 1 {
150+
t.Fatalf("unexpected number of envionments: %d != %d", len(envs), 1)
151+
}
152+
if envs[0].IntroducedIn.String() != hashes[2].String() {
153+
t.Fatalf("unexpected introducedIn: %s != %s", envs[0].IntroducedIn.String(), hashes[2].String())
154+
}
155+
}

‎nodejs/coalescer_test.go

-71
This file was deleted.

‎nodejs/ecosystem.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
"github.com/quay/claircore/indexer"
7+
"github.com/quay/claircore/language"
78
)
89

910
var scanners = []indexer.PackageScanner{&Scanner{}}
@@ -14,6 +15,6 @@ func NewEcosystem(_ context.Context) *indexer.Ecosystem {
1415
PackageScanners: func(_ context.Context) ([]indexer.PackageScanner, error) { return scanners, nil },
1516
DistributionScanners: func(_ context.Context) ([]indexer.DistributionScanner, error) { return nil, nil },
1617
RepositoryScanners: func(_ context.Context) ([]indexer.RepositoryScanner, error) { return nil, nil },
17-
Coalescer: NewCoalescer,
18+
Coalescer: language.NewCoalescer,
1819
}
1920
}

0 commit comments

Comments
 (0)