diff --git a/pkg/ingestor/parser/common/scanner/scanner.go b/pkg/ingestor/parser/common/scanner/scanner.go index 077abbf14f..a509a36422 100644 --- a/pkg/ingestor/parser/common/scanner/scanner.go +++ b/pkg/ingestor/parser/common/scanner/scanner.go @@ -19,8 +19,11 @@ import ( "context" "fmt" "net/http" + "time" + deps_dev_client "github.com/guacsec/guac/internal/client/depsdevclient" "github.com/guacsec/guac/pkg/assembler" + "github.com/guacsec/guac/pkg/assembler/clients/generated" cd_certifier "github.com/guacsec/guac/pkg/certifier/clearlydefined" eol_certifier "github.com/guacsec/guac/pkg/certifier/eol" osv_certifier "github.com/guacsec/guac/pkg/certifier/osv" @@ -114,8 +117,68 @@ func purlsLicenseScanWithClient( return certLegalIngest, hasSourceAtIngest, nil } +// PurlDepsDevScan scans the purls and returns for metadata linked to the repository +// and returns the list of scorecards and sources associations it finds from Deps.dev +// This generally takes about 300ms - 600ms. With tests including 1-30 PURLs. func PurlsDepsDevScan(ctx context.Context, purls []string) ([]assembler.CertifyScorecardIngest, []assembler.HasSourceAtIngest, error) { - return nil, nil, fmt.Errorf("Unimplemented") + certifyScorecards := []assembler.CertifyScorecardIngest{} + hasSourceAts := []assembler.HasSourceAtIngest{} + + ddc, err := deps_dev_client.NewDepsClient(ctx) + if err != nil { + return nil, nil, fmt.Errorf("unable to initialize deps.dev client: %w", err) + + } + + ddc.RetrieveVersionsAndProjects(ctx, purls) + components, err := ddc.GetMetadata(ctx, purls) + if err != nil { + return nil, nil, fmt.Errorf("unable to get metadata for purls in deps.dev: %w", err) + } + + for _, c := range components { + hasSourceAt := createDepsDevHasSourceAtIngest(c.CurrentPackage, c.Source, c.UpdateTime.UTC()) + scorecard := createDepsDevScorecardIngest(c.Source, c.Scorecard) + if hasSourceAt != nil { + hasSourceAts = append(hasSourceAts, *hasSourceAt) + } + if scorecard != nil { + certifyScorecards = append(certifyScorecards, *scorecard) + } + } + return certifyScorecards, hasSourceAts, nil +} + +func createDepsDevHasSourceAtIngest(pkg *generated.PkgInputSpec, src *generated.SourceInputSpec, knownSince time.Time) *assembler.HasSourceAtIngest { + if pkg != nil && src != nil { + return &assembler.HasSourceAtIngest{ + Pkg: pkg, + PkgMatchFlag: generated.MatchFlags{ + Pkg: generated.PkgMatchTypeAllVersions, + }, + Src: src, + HasSourceAt: &generated.HasSourceAtInputSpec{ + KnownSince: knownSince, + Justification: "collected via deps.dev", + Collector: "ingest_depsdev_scanner", + Origin: "deps.dev", + }, + } + } + return nil +} + +func createDepsDevScorecardIngest(src *generated.SourceInputSpec, scorecard *generated.ScorecardInputSpec) *assembler.CertifyScorecardIngest { + if src != nil && scorecard != nil { + scorecard.Origin = "deps.dev" + scorecard.Collector = "ingest_depsdev_scanner" + + return &assembler.CertifyScorecardIngest{ + Source: src, + Scorecard: scorecard, + } + } + return nil } // runQueryOnBatchedPurls runs EvaluateClearlyDefinedDefinition from the clearly defined diff --git a/pkg/ingestor/parser/common/scanner/scanner_test.go b/pkg/ingestor/parser/common/scanner/scanner_test.go index 2623f2f882..1814895b19 100644 --- a/pkg/ingestor/parser/common/scanner/scanner_test.go +++ b/pkg/ingestor/parser/common/scanner/scanner_test.go @@ -700,3 +700,315 @@ func parseEOLValue(value string) map[string]string { } return result } + +func TestPurlsDepsDevScan(t *testing.T) { + ctx := logging.WithLogger(context.Background()) + tm, _ := time.Parse(time.RFC3339, "2022-11-21T17:45:50.52Z") + tests := []struct { + name string + purls []string + wantCSs []assembler.CertifyScorecardIngest + wantHSAs []assembler.HasSourceAtIngest + wantErr bool + }{ + { + name: "no purl", + purls: []string{""}, + wantCSs: []assembler.CertifyScorecardIngest{}, + wantHSAs: []assembler.HasSourceAtIngest{}, + wantErr: false, + }, + { + name: "valid maven/org.webjars.npm/a", + purls: []string{"pkg:maven/org.webjars.npm/a@2.1.2"}, + wantCSs: []assembler.CertifyScorecardIngest{ + { + Source: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/alfateam", + Name: "a", + }, + Scorecard: &generated.ScorecardInputSpec{ + // Not including all checks since they are not stable for testing + // and are not used anyway. As long as its a non-empty list. + Checks: []generated.ScorecardCheckInputSpec{ + { + Check: "Pinned-Dependencies", + Score: -1, + }, + { + Check: "Token-Permissions", + Score: -1, + }, + { + Check: "Maintained", + Score: 0, + }, + { + Check: "Binary-Artifacts", + Score: 10, + }, + }, + AggregateScore: 3.0999999046325684, + TimeScanned: tm, + ScorecardVersion: "v5.0.0-94-g51f31c98", + ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantHSAs: []assembler.HasSourceAtIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "maven", + Namespace: ptrfrom.String("org.webjars.npm"), + Name: "a", + Version: ptrfrom.String("2.1.2"), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"}, + Src: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/alfateam", + Name: "a", + }, + HasSourceAt: &generated.HasSourceAtInputSpec{ + KnownSince: tm, + Justification: "collected via deps.dev", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantErr: false, + }, + { + name: "non-existent purl", + purls: []string{"pkg:notexist/ns/a@1.2"}, + wantCSs: []assembler.CertifyScorecardIngest{}, + wantHSAs: []assembler.HasSourceAtIngest{}, + wantErr: false, + }, + { + name: "malformed purl", + purls: []string{"not-a-purl"}, + wantCSs: []assembler.CertifyScorecardIngest{}, + wantHSAs: []assembler.HasSourceAtIngest{}, + wantErr: false, + }, + { + name: "valid pypi/wheel-axle-runtime", + purls: []string{"pkg:pypi/wheel-axle-runtime@0.0.4"}, + wantCSs: []assembler.CertifyScorecardIngest{ + { + Source: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/karellen", + Name: "wheel-axle-runtime", + }, + Scorecard: &generated.ScorecardInputSpec{ + // Not including all checks since they are not stable for testing + // and are not used anyway. As long as its a non-empty list. + Checks: []generated.ScorecardCheckInputSpec{ + { + Check: "Pinned-Dependencies", + Score: 0, + }, + { + Check: "Token-Permissions", + Score: 8, + }, + { + Check: "Maintained", + Score: 1, + }, + { + Check: "Binary-Artifacts", + Score: 8, + }, + }, + AggregateScore: 3.700000047683716, + TimeScanned: tm, + ScorecardVersion: "v5.0.0-94-g51f31c98", + ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantHSAs: []assembler.HasSourceAtIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "pypi", + Namespace: ptrfrom.String(""), + Name: "wheel-axle-runtime", + Version: ptrfrom.String("0.0.4"), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"}, + Src: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/karellen", + Name: "wheel-axle-runtime", + }, + HasSourceAt: &generated.HasSourceAtInputSpec{ + KnownSince: tm, + Justification: "collected via deps.dev", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantErr: false, + }, + { + name: "multiple valid PURLs", + purls: []string{ + "pkg:maven/org.webjars.npm/a@2.1.2", + "pkg:pypi/wheel-axle-runtime@0.0.4", + }, + wantCSs: []assembler.CertifyScorecardIngest{ + { + Source: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/alfateam", + Name: "a", + }, + Scorecard: &generated.ScorecardInputSpec{ + // Not including all checks since they are not stable for testing + // and are not used anyway. As long as its a non-empty list. + Checks: []generated.ScorecardCheckInputSpec{ + { + Check: "Pinned-Dependencies", + Score: -1, + }, + { + Check: "Token-Permissions", + Score: -1, + }, + { + Check: "Maintained", + Score: 0, + }, + { + Check: "Binary-Artifacts", + Score: 10, + }, + }, + AggregateScore: 3.0999999046325684, + TimeScanned: tm, + ScorecardVersion: "v5.0.0-94-g51f31c98", + ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + { + Source: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/karellen", + Name: "wheel-axle-runtime", + }, + Scorecard: &generated.ScorecardInputSpec{ + // Not including all checks since they are not stable for testing + // and are not used anyway. As long as its a non-empty list. + Checks: []generated.ScorecardCheckInputSpec{ + { + Check: "Pinned-Dependencies", + Score: 0, + }, + { + Check: "Token-Permissions", + Score: 8, + }, + { + Check: "Maintained", + Score: 1, + }, + { + Check: "Binary-Artifacts", + Score: 8, + }, + }, + AggregateScore: 3.700000047683716, + TimeScanned: tm, + ScorecardVersion: "v5.0.0-94-g51f31c98", + ScorecardCommit: "51f31c9882b6e5998e0df571096147a99842092b", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantHSAs: []assembler.HasSourceAtIngest{ + { + Pkg: &generated.PkgInputSpec{ + Type: "maven", + Namespace: ptrfrom.String("org.webjars.npm"), + Name: "a", + Version: ptrfrom.String("2.1.2"), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"}, + Src: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/alfateam", + Name: "a", + }, + HasSourceAt: &generated.HasSourceAtInputSpec{ + KnownSince: tm, + Justification: "collected via deps.dev", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + { + Pkg: &generated.PkgInputSpec{ + Type: "pypi", + Namespace: ptrfrom.String(""), + Name: "wheel-axle-runtime", + Version: ptrfrom.String("0.0.4"), + Subpath: ptrfrom.String(""), + }, + PkgMatchFlag: generated.MatchFlags{Pkg: "ALL_VERSIONS"}, + Src: &generated.SourceInputSpec{ + Type: "git", + Namespace: "github.com/karellen", + Name: "wheel-axle-runtime", + }, + HasSourceAt: &generated.HasSourceAtInputSpec{ + KnownSince: tm, + Justification: "collected via deps.dev", + Origin: "deps.dev", + Collector: "ingest_depsdev_scanner", + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCSs, gotHSAs, err := PurlsDepsDevScan(ctx, tt.purls) + if (err != nil) != tt.wantErr { + t.Errorf("PurlsToScan() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff( + tt.wantCSs, + gotCSs, + cmpopts.IgnoreFields(generated.ScorecardInputSpec{}, + "AggregateScore", "TimeScanned", + "ScorecardVersion", "ScorecardCommit", "DocumentRef"), + cmpopts.IgnoreTypes(generated.ScorecardCheckInputSpec{}), + ); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + if diff := cmp.Diff(tt.wantHSAs, gotHSAs, + cmpopts.IgnoreFields(generated.HasSourceAtInputSpec{}, "KnownSince", "DocumentRef"), + ); diff != "" { + t.Errorf("Unexpected results. (-want +got):\n%s", diff) + } + }) + } +}