feat(scanner): add keppel scanner

scanner/keppel/go.mod

@@ -1,3 +1,24 @@
 module github.com/cloudoperators/heureka/scanners/keppel
 
-go 1.22.6
+go 1.22.6
+
+require (
+	github.com/aquasecurity/trivy v0.54.1
+	github.com/gophercloud/gophercloud v1.14.0
+	github.com/gophercloud/gophercloud/v2 v2.1.0
+	github.com/kelseyhightower/envconfig v1.4.0
+	github.com/pkg/errors v0.9.1
+)
+
+require golang.org/x/sys v0.22.0 // indirect
+
+require (
+	github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 // indirect
+	github.com/google/go-containerregistry v0.20.1 // indirect
+	github.com/machinebox/graphql v0.2.2
+	github.com/package-url/packageurl-go v0.1.3 // indirect
+	github.com/samber/lo v1.46.0 // indirect
+	github.com/sirupsen/logrus v1.9.3
+	golang.org/x/text v0.16.0 // indirect
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+)

scanner/keppel/go.sum

@@ -0,0 +1,45 @@
+github.com/aquasecurity/trivy v0.54.1 h1:/uNCF06PfdC69v5n3Zh4fXVf0xmXBml0c/ergf066SQ=
+github.com/aquasecurity/trivy v0.54.1/go.mod h1:i5S54WUtOEN9egFF0AHsxq6XT7QD11n9pSmIXhMJV0g=
+github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04 h1:6/T8sFdNVG/AwOGoK6X55h7hF7LYqK8bsuPz8iEz8jM=
+github.com/aquasecurity/trivy-db v0.0.0-20240718084044-d23a6ca8ba04/go.mod h1:0T6oy2t1Iedt+yi3Ml5cpOYp5FZT4MI1/mx+3p+PIs8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-containerregistry v0.20.1 h1:eTgx9QNYugV4DN5mz4U8hiAGTi1ybXn0TPi4Smd8du0=
+github.com/google/go-containerregistry v0.20.1/go.mod h1:YCMFNQeeXeLF+dnhhWkqDItx/JSkH01j1Kis4PsjzFI=
+github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8=
+github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM=
+github.com/gophercloud/gophercloud/v2 v2.1.0 h1:91p6c+uMckXyx39nSIYjDirDBnPVFQq0q1njLNPX+NY=
+github.com/gophercloud/gophercloud/v2 v2.1.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo=
+github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAEolaQybkjWA=
+github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs=
+github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/samber/lo v1.46.0 h1:w8G+oaCPgz1PoCJztqymCFaKwXt+5cCXn51uPxExFfQ=
+github.com/samber/lo v1.46.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
+golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

scanner/keppel/main.go

@@ -4,9 +4,125 @@
 package main
 
 import (
-	"fmt"
+	"os"
+	"sync"
+
+	"github.com/cloudoperators/heureka/scanners/keppel/models"
+	"github.com/cloudoperators/heureka/scanners/keppel/processor"
+	"github.com/cloudoperators/heureka/scanners/keppel/scanner"
+	"github.com/kelseyhightower/envconfig"
+	log "github.com/sirupsen/logrus"
 )
 
+func init() {
+	// Log as JSON instead of the default ASCII formatter.
+	log.SetFormatter(&log.JSONFormatter{})
+
+	// Output to stdout instead of the default stderr
+	// Can be any io.Writer, see below for File example
+	log.SetOutput(os.Stdout)
+
+	// Only log the warning severity or above.
+	log.SetLevel(log.DebugLevel)
+}
+
 func main() {
-	fmt.Println("This is the Keppel scanner")
+	var wg sync.WaitGroup
+	var scannerCfg scanner.Config
+	err := envconfig.Process("heureka", &scannerCfg)
+	if err != nil {
+		log.WithError(err).Fatal("Error while reading env config for scanner")
+	}
+
+	var processorCfg processor.Config
+	err = envconfig.Process("heureka", &processorCfg)
+	if err != nil {
+		log.WithError(err).Fatal("Error while reading env config for processor")
+	}
+
+	keppelScanner := scanner.NewScanner(scannerCfg)
+	keppelProcessor := processor.NewProcessor(processorCfg)
+
+	err = keppelScanner.Setup()
+	if err != nil {
+		log.WithError(err).Fatal("Error during scanner setup")
+	}
+	accounts, err := keppelScanner.ListAccounts()
+	if err != nil {
+		log.WithError(err).Fatal("Error during ListAccounts")
+	}
+
+	wg.Add(len(accounts))
+
+	for _, account := range accounts {
+		go ProcessAccount(account, keppelScanner, keppelProcessor, &wg)
+	}
+
+	wg.Wait()
+}
+
+func ProcessAccount(account models.Account, keppelScanner *scanner.Scanner, keppelProcessor *processor.Processor, wg *sync.WaitGroup) error {
+	defer wg.Done()
+	repositories, err := keppelScanner.ListRepositories(account.Name)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"account:": account.Name,
+		}).WithError(err).Error("Error during ProcessRepository")
+		return err
+	}
+
+	for _, repository := range repositories {
+		component, err := keppelProcessor.ProcessRepository(repository)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"account:":   account.Name,
+				"repository": repository.Name,
+			}).WithError(err).Error("Error during ProcessRepository")
+			componentPtr, err := keppelProcessor.GetComponent(repository.Name)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"account:":   account.Name,
+					"repository": repository.Name,
+				}).WithError(err).Error("Error during GetComponent")
+			}
+			component = *componentPtr
+		}
+
+		manifests, err := keppelScanner.ListManifests(account.Name, repository.Name)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"account:":   account.Name,
+				"repository": repository.Name,
+			}).WithError(err).Error("Error during ListManifests")
+			continue
+		}
+		for _, manifest := range manifests {
+			componentVersion, err := keppelProcessor.ProcessManifest(manifest, component.ID)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"account:":   account.Name,
+					"repository": repository.Name,
+				}).WithError(err).Error("Error during ProcessManifest")
+				componentVersionPtr, err := keppelProcessor.GetComponentVersion(manifest.Digest)
+				if err != nil {
+					log.WithFields(log.Fields{
+						"account:":   account.Name,
+						"repository": repository.Name,
+					}).WithError(err).Error("Error during GetComponentVersion")
+				}
+				componentVersion = *componentVersionPtr
+			}
+			trivyReport, err := keppelScanner.GetTrivyReport(account.Name, repository.Name, manifest.Digest)
+			if err != nil {
+				log.WithFields(log.Fields{
+					"account:":   account.Name,
+					"repository": repository.Name,
+				}).WithError(err).Error("Error during GetTrivyReport")
+				continue
+			}
+			keppelProcessor.ProcessReport(*trivyReport, componentVersion.ID)
+		}
+	}
+
+	return nil
 }

scanner/keppel/models/models.go

@@ -0,0 +1,130 @@
+// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package models
+
+import (
+	"time"
+
+	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
+	stypes "github.com/aquasecurity/trivy/pkg/module/serialize"
+)
+
+// For more attributes check
+// https://github.com/sapcc/keppel/blob/master/internal/models/account.go
+type Account struct {
+	Name         string
+	AuthTenantID string `json:"auth_tenant_id"`
+}
+
+type AccountResponse struct {
+	Accounts []Account
+}
+
+type RepositoryResponse struct {
+	Repositories []Repository
+}
+
+type Repository struct {
+	Name          string `json:"name"`
+	ManifestCount uint64 `json:"manifest_count"`
+	TagCount      uint64 `json:"tag_count"`
+	SizeBytes     uint64 `json:"size_bytes,omitempty"`
+	PushedAt      int64  `json:"pushed_at,omitempty"`
+}
+
+type Manifest struct {
+	Digest              string
+	MediaType           string `json:"media_type"`
+	SizeBytes           uint64 `json:"size_bytes"`
+	PushedAt            uint64 `json:"pushed_at"`
+	LastPulledAt        uint64 `json:"last_pulled_at"`
+	VulnerabilityStatus string `json:"vulnerability_status"`
+	Labels              Labels
+	MaxLayerCreatedAt   int64 `json:"max_layer_created_at"`
+	MinLayerCreatedAt   int64 `json:"min_layer_created_at"`
+}
+
+type ManifestResponse struct {
+	Manifests []Manifest
+}
+
+type Labels struct {
+	Maintainer       string
+	Maintainers      string
+	SourceRepository string `json:"source_repository"`
+}
+
+type TrivyReport struct {
+	SchemaVersion int            `json:",omitempty"`
+	CreatedAt     time.Time      `json:",omitempty"`
+	ArtifactName  string         `json:",omitempty"`
+	ArtifactType  string         `json:",omitempty"` // generic replacement for original type `artifact.Type`
+	Metadata      TrivyMetadata  `json:",omitempty"` // generic replacement for original type `types.Metadata`
+	Results       stypes.Results `json:",omitempty"` // compatible replacement for original type `types.Results`
+}
+
+type TrivyMetadata struct {
+	Size int64      `json:",omitempty"`
+	OS   *ftypes.OS `json:",omitempty"`
+
+	// Container image
+	ImageID     string         `json:",omitempty"`
+	DiffIDs     []string       `json:",omitempty"`
+	RepoTags    []string       `json:",omitempty"`
+	RepoDigests []string       `json:",omitempty"`
+	ImageConfig map[string]any `json:",omitempty"`
+}
+
+type Component struct {
+	ID   string  `json:"id"`
+	Name *string `json:"name,omitempty"`
+	Type *string `json:"type,omitempty"`
+}
+
+type ComponentConnection struct {
+	TotalCount int              `json:"totalCount"`
+	Edges      []*ComponentEdge `json:"edges,omitempty"`
+}
+
+type ComponentEdge struct {
+	Node   *Component `json:"node"`
+	Cursor *string    `json:"cursor,omitempty"`
+}
+
+type ComponentVersion struct {
+	ID          string  `json:"id"`
+	Version     *string `json:"version,omitempty"`
+	ComponentID *string `json:"componentId,omitempty"`
+}
+
+type ComponentVersionConnection struct {
+	TotalCount int                     `json:"totalCount"`
+	Edges      []*ComponentVersionEdge `json:"edges"`
+}
+
+type ComponentVersionEdge struct {
+	Node   *ComponentVersion `json:"node"`
+	Cursor *string           `json:"cursor,omitempty"`
+}
+
+type IssueConnection struct {
+	TotalCount           int          `json:"totalCount"`
+	VulnerabilityCount   int          `json:"vulnerabilityCount"`
+	PolicyViolationCount int          `json:"policyViolationCount"`
+	SecurityEventCount   int          `json:"securityEventCount"`
+	Edges                []*IssueEdge `json:"edges"`
+}
+
+type IssueEdge struct {
+	Node   *Issue  `json:"node"`
+	Cursor *string `json:"cursor,omitempty"`
+}
+
+type Issue struct {
+	ID           string  `json:"id"`
+	Type         *string `json:"type,omitempty"`
+	PrimaryName  *string `json:"primaryName,omitempty"`
+	Description  *string `json:"description,omitempty"`
+	LastModified *string `json:"lastModified,omitempty"`
+}

scanner/keppel/processor/config.go

@@ -0,0 +1,8 @@
+// SPDX-FileCopyrightText: 2024 SAP SE or an SAP affiliate company and Greenhouse contributors
+// SPDX-License-Identifier: Apache-2.0
+
+package processor
+
+type Config struct {
+	HeurekaUrl string `envconfig:"HEUREKA_URL" required:"true" json:"-"`
+}