diff --git a/README.md b/README.md index 7477d30ae8..6f089d1446 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ The following are [GraphQL backends](pkg/assembler/backends) that are implemente 3. Optimized: The backend has gone through a level of optimization to help improve performance. The enumerated backends are: -- [inmem (supported, complete, optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/inmem): a non-persistent in-memory backend that doesn't require any additional infrastructure. Also acts as a conformance backend for API implementations. We recommend starting with this if you're just starting with GUAC! +- [keyvalue (supported, complete, optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/keyvalue): a non-persistent in-memory backend that doesn't require any additional infrastructure. Also acts as a conformance backend for API implementations. We recommend starting with this if you're just starting with GUAC! - [arangoDB (supported, incomplete, optimized)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/arangodb): a persistent backend based on [ArangoDB](https://arangodb.com/) - [ent (supported, incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/ent): a persistent backend based on [ent](https://entgo.io/) that can run on various SQL backends such as [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/) and [SQLite](https://www.sqlite.org/index.html). - [neo4j/openCypher (unsupported, incomplete)](https://github.com/guacsec/guac/tree/main/pkg/assembler/backends/neo4j): a persistent backend based on [neo4j](https://neo4j.com/) and [openCypher](https://opencypher.org/). This backend should work with any database that supported openCypher queries. diff --git a/cmd/README.md b/cmd/README.md index 0a2a4b6af8..5d7530253f 100644 --- a/cmd/README.md +++ b/cmd/README.md @@ -32,7 +32,7 @@ services: - what it does: runs a GraphQL server - options: - - backend: inmem, neo4j, arango, ent, or future DB + - backend: keyvalue, neo4j, arango, ent, or future DB - backend-specific options: neo4j connection options - playground / debug: also start playground diff --git a/cmd/guacgql/cmd/server.go b/cmd/guacgql/cmd/server.go index a7636bb93b..15754eb92b 100644 --- a/cmd/guacgql/cmd/server.go +++ b/cmd/guacgql/cmd/server.go @@ -30,7 +30,7 @@ import ( "github.com/99designs/gqlgen/graphql/playground" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/backends/arangodb" - _ "github.com/guacsec/guac/pkg/assembler/backends/inmem" + _ "github.com/guacsec/guac/pkg/assembler/backends/keyvalue" "github.com/guacsec/guac/pkg/assembler/backends/neo4j" "github.com/guacsec/guac/pkg/assembler/backends/neptune" "github.com/guacsec/guac/pkg/assembler/graphql/generated" @@ -43,9 +43,9 @@ import ( const ( arango = "arango" neo4js = "neo4j" - inmems = "inmem" ent = "ent" neptunes = "neptune" + keyvalue = "keyvalue" ) type optsFunc func() backends.BackendArgs @@ -58,8 +58,8 @@ func init() { } getOpts[arango] = getArango getOpts[neo4js] = getNeo4j - getOpts[inmems] = getInMem getOpts[neptunes] = getNeptune + getOpts[keyvalue] = getKeyValue } func startServer(cmd *cobra.Command) { @@ -171,7 +171,7 @@ func getNeo4j() backends.BackendArgs { } } -func getInMem() backends.BackendArgs { +func getKeyValue() backends.BackendArgs { return nil } diff --git a/container_files/guac/guac.yaml b/container_files/guac/guac.yaml index 737f5abf27..bba73c16e5 100644 --- a/container_files/guac/guac.yaml +++ b/container_files/guac/guac.yaml @@ -6,7 +6,7 @@ csub-addr: guac-collectsub:2782 csub-listen-port: 2782 # graphql -gql-backend: inmem +gql-backend: keyvalue gql-listen-port: 8080 gql-debug: true gql-addr: http://guac-graphql:8080/query @@ -17,4 +17,4 @@ use-csub: true # certifier polling poll: true -interval: 5m \ No newline at end of file +interval: 5m diff --git a/guac.yaml b/guac.yaml index 7b5570326c..c0d25fa554 100644 --- a/guac.yaml +++ b/guac.yaml @@ -25,7 +25,7 @@ csub-addr: localhost:2782 csub-listen-port: 2782 # GQL setup -gql-backend: inmem +gql-backend: keyvalue gql-listen-port: 8080 gql-debug: true gql-addr: http://localhost:8080/query diff --git a/internal/testing/stablememmap/stablememmap.go b/internal/testing/stablememmap/stablememmap.go new file mode 100644 index 0000000000..b3287e0dc4 --- /dev/null +++ b/internal/testing/stablememmap/stablememmap.go @@ -0,0 +1,51 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package stablememmap + +import ( + "context" + "slices" + + "github.com/guacsec/guac/pkg/assembler/kv" + "github.com/guacsec/guac/pkg/assembler/kv/memmap" +) + +type Store struct { + mm kv.Store +} + +func GetStore() kv.Store { + return &Store{ + mm: memmap.GetStore(), + } +} + +func (s *Store) Get(ctx context.Context, c, k string, v any) error { + return s.mm.Get(ctx, c, k, v) +} + +func (s *Store) Set(ctx context.Context, c, k string, v any) error { + return s.mm.Set(ctx, c, k, v) +} + +func (s *Store) Keys(ctx context.Context, c string) ([]string, error) { + keys, err := s.mm.Keys(ctx, c) + if err != nil { + return nil, err + } + slices.Sort(keys) + return keys, nil +} diff --git a/pkg/assembler/backends/inmem/artifact.go b/pkg/assembler/backends/inmem/artifact.go deleted file mode 100644 index 57faadd3e7..0000000000 --- a/pkg/assembler/backends/inmem/artifact.go +++ /dev/null @@ -1,266 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// Internal data: Artifacts -type artMap map[string]*artStruct -type artStruct struct { - id uint32 - algorithm string - digest string - hashEquals []uint32 - occurrences []uint32 - hasSBOMs []uint32 - hasSLSAs []uint32 - vexLinks []uint32 - badLinks []uint32 - goodLinks []uint32 - hasMetadataLinks []uint32 - pointOfContactLinks []uint32 -} - -func (n *artStruct) ID() uint32 { return n.id } - -func (n *artStruct) Neighbors(allowedEdges edgeMap) []uint32 { - out := []uint32{} - if allowedEdges[model.EdgeArtifactHashEqual] { - out = append(out, n.hashEquals...) - } - if allowedEdges[model.EdgeArtifactIsOccurrence] { - out = append(out, n.occurrences...) - } - if allowedEdges[model.EdgeArtifactHasSbom] { - out = append(out, n.hasSBOMs...) - } - if allowedEdges[model.EdgeArtifactHasSlsa] { - out = append(out, n.hasSLSAs...) - } - if allowedEdges[model.EdgeArtifactCertifyVexStatement] { - out = append(out, n.vexLinks...) - } - if allowedEdges[model.EdgeArtifactCertifyBad] { - out = append(out, n.badLinks...) - } - if allowedEdges[model.EdgeArtifactCertifyGood] { - out = append(out, n.goodLinks...) - } - if allowedEdges[model.EdgeArtifactHasMetadata] { - out = append(out, n.hasMetadataLinks...) - } - if allowedEdges[model.EdgeArtifactPointOfContact] { - out = append(out, n.pointOfContactLinks...) - } - - return out -} - -func (n *artStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convArtifact(n), nil -} - -func (n *artStruct) setHashEquals(id uint32) { n.hashEquals = append(n.hashEquals, id) } -func (n *artStruct) setOccurrences(id uint32) { n.occurrences = append(n.occurrences, id) } -func (n *artStruct) setHasSBOMs(id uint32) { n.hasSBOMs = append(n.hasSBOMs, id) } -func (n *artStruct) setHasSLSAs(id uint32) { n.hasSLSAs = append(n.hasSLSAs, id) } -func (n *artStruct) setVexLinks(id uint32) { n.vexLinks = append(n.vexLinks, id) } -func (n *artStruct) setCertifyBadLinks(id uint32) { n.badLinks = append(n.badLinks, id) } -func (n *artStruct) setCertifyGoodLinks(id uint32) { n.goodLinks = append(n.goodLinks, id) } -func (n *artStruct) setHasMetadataLinks(id uint32) { - n.hasMetadataLinks = append(n.hasMetadataLinks, id) -} -func (n *artStruct) setPointOfContactLinks(id uint32) { - n.pointOfContactLinks = append(n.pointOfContactLinks, id) -} - -// Ingest Artifacts - -func (c *demoClient) IngestArtifacts(ctx context.Context, artifacts []*model.ArtifactInputSpec) ([]*model.Artifact, error) { - var modelArtifacts []*model.Artifact - for _, art := range artifacts { - modelArt, err := c.IngestArtifact(ctx, art) - if err != nil { - return nil, gqlerror.Errorf("ingestArtifact failed with err: %v", err) - } - modelArtifacts = append(modelArtifacts, modelArt) - } - return modelArtifacts, nil -} - -func (c *demoClient) IngestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec) (*model.Artifact, error) { - return c.ingestArtifact(ctx, artifact, true) -} - -func (c *demoClient) ingestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec, readOnly bool) (*model.Artifact, error) { - algorithm := strings.ToLower(artifact.Algorithm) - digest := strings.ToLower(artifact.Digest) - - lock(&c.m, readOnly) - defer unlock(&c.m, readOnly) - - a, err := c.artifactByKey(algorithm, digest) - if err != nil { - if readOnly { - c.m.RUnlock() - a, err := c.ingestArtifact(ctx, artifact, false) - c.m.RLock() // relock so that defer unlock does not panic - return a, err - } - a = &artStruct{ - id: c.getNextID(), - algorithm: algorithm, - digest: digest, - } - c.index[a.id] = a - c.artifacts[strings.Join([]string{algorithm, digest}, ":")] = a - } - - return c.convArtifact(a), nil -} - -func (c *demoClient) artifactByKey(alg, dig string) (*artStruct, error) { - algorithm := strings.ToLower(alg) - digest := strings.ToLower(dig) - if a, ok := c.artifacts[strings.Join([]string{algorithm, digest}, ":")]; ok { - return a, nil - } - return nil, errors.New("artifact not found") -} - -func (c *demoClient) artifactExact(artifactSpec *model.ArtifactSpec) (*artStruct, error) { - algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm)) - digest := strings.ToLower(nilToEmpty(artifactSpec.Digest)) - - // If ID is provided, try to look up, then check if algo and digest match. - if artifactSpec.ID != nil { - id64, err := strconv.ParseUint(*artifactSpec.ID, 10, 32) - if err != nil { - return nil, fmt.Errorf("couldn't parse id %w", err) - } - id := uint32(id64) - a, err := byID[*artStruct](id, c) - if err != nil { - // Not found - return nil, nil - } - // If found by id, ignore rest of fields in spec and return as a match - return a, nil - } - - // If algo and digest are provided, try to lookup - if algorithm != "" && digest != "" { - if a, err := c.artifactByKey(algorithm, digest); err != nil { - return a, nil - } - } - return nil, nil -} - -// Query Artifacts - -func (c *demoClient) Artifacts(ctx context.Context, artifactSpec *model.ArtifactSpec) ([]*model.Artifact, error) { - c.m.RLock() - defer c.m.RUnlock() - a, err := c.artifactExact(artifactSpec) - if err != nil { - return nil, gqlerror.Errorf("Artifacts :: invalid spec %s", err) - } - if a != nil { - return []*model.Artifact{c.convArtifact(a)}, nil - } - - algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm)) - digest := strings.ToLower(nilToEmpty(artifactSpec.Digest)) - var rv []*model.Artifact - for _, a := range c.artifacts { - matchAlgorithm := false - if algorithm == "" || algorithm == a.algorithm { - matchAlgorithm = true - } - - matchDigest := false - if digest == "" || digest == a.digest { - matchDigest = true - } - - if matchDigest && matchAlgorithm { - rv = append(rv, c.convArtifact(a)) - } - } - return rv, nil -} - -func (c *demoClient) convArtifact(a *artStruct) *model.Artifact { - return &model.Artifact{ - ID: nodeID(a.id), - Digest: a.digest, - Algorithm: a.algorithm, - } -} - -// Builds a model.Artifact to send as GraphQL response, starting from id. -// The optional filter allows restricting output (on selection operations). -func (c *demoClient) buildArtifactResponse(id uint32, filter *model.ArtifactSpec) (*model.Artifact, error) { - if filter != nil && filter.ID != nil { - filteredID, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - if uint32(filteredID) != id { - return nil, nil - } - } - - artNode, err := byID[*artStruct](id, c) - if err != nil { - return nil, fmt.Errorf("ID does not match expected node type for artifact, %w", err) - } - - if filter != nil && noMatch(toLower(filter.Algorithm), artNode.algorithm) { - return nil, nil - } - if filter != nil && noMatch(toLower(filter.Digest), artNode.digest) { - return nil, nil - } - art := &model.Artifact{ - // IDs are generated as string even though we ask for integers - // See https://github.com/99designs/gqlgen/issues/2561 - ID: nodeID(artNode.id), - Algorithm: artNode.algorithm, - Digest: artNode.digest, - } - - return art, nil -} - -func getArtifactIDFromInput(c *demoClient, input model.ArtifactInputSpec) (uint32, error) { - a, err := c.artifactByKey(input.Algorithm, input.Digest) - if err != nil { - return 0, gqlerror.Errorf("artifact with algorithm \"%s\" and digest \"%s\" not found", input.Algorithm, input.Digest) - } - return a.id, nil -} diff --git a/pkg/assembler/backends/inmem/backend.go b/pkg/assembler/backends/inmem/backend.go deleted file mode 100644 index 924f9393bc..0000000000 --- a/pkg/assembler/backends/inmem/backend.go +++ /dev/null @@ -1,265 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "errors" - "fmt" - "math" - "reflect" - "slices" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/guacsec/guac/pkg/assembler/backends" - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -func init() { - backends.Register("inmem", getBackend) -} - -// node is the common interface of all backend nodes. -type node interface { - // ID provides global IDs for all nodes that can be referenced from - // other places in GUAC. - // - // Since we always ingest data and never remove, - // we can keep this global and increment it as needed. - // - // For fast retrieval, we also keep a map from ID from nodes that have - // it. - // - // IDs are stored as string in graphql even though we ask for integers - // See https://github.com/99designs/gqlgen/issues/2561 - ID() uint32 - - // Neighbors allows retrieving neighbors of a node using the backlinks. - // - // This is useful for path related queries where the type of the node - // is not as relevant as its connections. - // - // The allowedEdges argument allows filtering the set of neighbors to - // only include certain GUAC verbs. - Neighbors(allowedEdges edgeMap) []uint32 - - // BuildModelNode builds a GraphQL return type for a backend node, - BuildModelNode(c *demoClient) (model.Node, error) -} - -type indexType map[uint32]node - -var errNotFound = errors.New("not found") - -// Scorecard scores are in range of 1-10, so a single step at 100 should be -// plenty big -var epsilon = math.Nextafter(100, 100.1) - 100 - -// atomic add to ensure ID is not duplicated -func (c *demoClient) getNextID() uint32 { - return atomic.AddUint32(&c.id, 1) -} - -type demoClient struct { - id uint32 - m sync.RWMutex - index indexType - - artifacts artMap - builders builderMap - licenses licMap - packages pkgTypeMap - sources srcTypeMap - vulnerabilities vulnTypeMap - - certifyBads badList - certifyGoods goodList - certifyLegals certifyLegalList - certifyVulnerabilities certifyVulnerabilityList - hasMetadatas hasMetadataList - hasSBOMs hasSBOMList - hasSLSAs hasSLSAList - hasSources hasSrcList - hashEquals hashEqualList - isDependencies isDependencyList - occurrences isOccurrenceList - pkgEquals pkgEqualList - pointOfContacts pointOfContactList - scorecards scorecardList - vexs vexList - vulnerabilityEquals vulnerabilityEqualList - vulnerabilityMetadatas vulnerabilityMetadataList -} - -func getBackend(_ context.Context, _ backends.BackendArgs) (backends.Backend, error) { - client := &demoClient{ - artifacts: artMap{}, - builders: builderMap{}, - index: indexType{}, - licenses: licMap{}, - packages: pkgTypeMap{}, - sources: srcTypeMap{}, - vulnerabilities: vulnTypeMap{}, - } - - return client, nil -} - -func nodeID(id uint32) string { - return fmt.Sprintf("%d", id) -} - -func noMatch(filter *string, value string) bool { - if filter != nil { - return value != *filter - } - return false -} - -func noMatchInput(filter *string, value string) bool { - if filter != nil { - return value != *filter - } - return value != "" -} - -func nilToEmpty(input *string) string { - if input == nil { - return "" - } - return *input -} - -func timePtrEqual(a, b *time.Time) bool { - if a == nil && b == nil { - return true - } - if a != nil && b != nil { - return a.Equal(*b) - } - return false -} - -func toLower(filter *string) *string { - if filter != nil { - lower := strings.ToLower(*filter) - return &lower - } - return nil -} - -func noMatchFloat(filter *float64, value float64) bool { - if filter != nil { - return math.Abs(*filter-value) > epsilon - } - return false -} - -func floatEqual(x float64, y float64) bool { - return math.Abs(x-y) < epsilon -} - -func byID[E node](id uint32, c *demoClient) (E, error) { - var nl E - o, ok := c.index[id] - if !ok { - return nl, fmt.Errorf("%w : id not in index", errNotFound) - } - s, ok := o.(E) - if !ok { - return nl, fmt.Errorf("%w : node not a %T", errNotFound, nl) - } - return s, nil -} - -func lock(m *sync.RWMutex, readOnly bool) { - if readOnly { - m.RLock() - } else { - m.Lock() - } -} - -func unlock(m *sync.RWMutex, readOnly bool) { - if readOnly { - m.RUnlock() - } else { - m.Unlock() - } -} - -func parseIDs(ids []string) ([]uint32, error) { - keys := make([]uint32, 0, len(ids)) - for _, id := range ids { - if key, err := parseID(id); err != nil { - return nil, err - } else { - keys = append(keys, key) - } - } - return keys, nil -} - -func parseID(id string) (uint32, error) { - id64, err := strconv.ParseUint(id, 10, 32) - return uint32(id64), err -} - -func sortAndRemoveDups(ids []uint32) []uint32 { - numIDs := len(ids) - if numIDs > 1 { - slices.Sort(ids) - nextIndex := 1 - for index := 1; index < numIDs; index++ { - currentVal := ids[index] - if ids[index-1] != currentVal { - ids[nextIndex] = currentVal - nextIndex++ - } - } - ids = ids[:nextIndex] - } - return ids -} - -func (c *demoClient) getPackageVersionAndArtifacts(pkgOrArt []uint32) (pkgs []uint32, arts []uint32, err error) { - for _, id := range pkgOrArt { - switch entry := c.index[id].(type) { - case *pkgVersionNode: - pkgs = append(pkgs, entry.id) - case *artStruct: - arts = append(arts, entry.id) - default: - return nil, nil, fmt.Errorf("unexpected type in package or artifact list: %s", reflect.TypeOf(entry)) - } - } - - return -} - -// IDs should be sorted -func (c *demoClient) isIDPresent(id string, linkIDs []uint32) bool { - linkID, err := strconv.ParseUint(id, 10, 32) - if err != nil { - return false - } - _, found := slices.BinarySearch[[]uint32](linkIDs, uint32(linkID)) - return found -} diff --git a/pkg/assembler/backends/inmem/certifyVEXStatement.go b/pkg/assembler/backends/inmem/certifyVEXStatement.go deleted file mode 100644 index 86767e7d59..0000000000 --- a/pkg/assembler/backends/inmem/certifyVEXStatement.go +++ /dev/null @@ -1,426 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "strconv" - "time" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// Internal data: link between a package or an artifact with its corresponding vulnerability VEX statement -type vexList []*vexLink -type vexLink struct { - id uint32 - packageID uint32 - artifactID uint32 - vulnerabilityID uint32 - knownSince time.Time - status model.VexStatus - statement string - statusNotes string - justification model.VexJustification - origin string - collector string -} - -func (n *vexLink) ID() uint32 { return n.id } - -func (n *vexLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2) - if n.packageID != 0 && allowedEdges[model.EdgeCertifyVexStatementPackage] { - out = append(out, n.packageID) - } - if n.artifactID != 0 && allowedEdges[model.EdgeCertifyVexStatementArtifact] { - out = append(out, n.artifactID) - } - if n.vulnerabilityID != 0 && allowedEdges[model.EdgeCertifyVexStatementVulnerability] { - out = append(out, n.vulnerabilityID) - } - return out -} - -func (n *vexLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildCertifyVEXStatement(n, nil, true) -} - -// Ingest CertifyVex - -func (c *demoClient) IngestVEXStatements(ctx context.Context, subjects model.PackageOrArtifactInputs, vulnerabilities []*model.VulnerabilityInputSpec, vexStatements []*model.VexStatementInputSpec) ([]string, error) { - var modelVexStatementIDs []string - - for i := range vexStatements { - var certVex *model.CertifyVEXStatement - var err error - if len(subjects.Packages) > 0 { - subject := model.PackageOrArtifactInput{Package: subjects.Packages[i]} - certVex, err = c.IngestVEXStatement(ctx, subject, *vulnerabilities[i], *vexStatements[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestVEXStatement failed with err: %v", err) - } - } else { - subject := model.PackageOrArtifactInput{Artifact: subjects.Artifacts[i]} - certVex, err = c.IngestVEXStatement(ctx, subject, *vulnerabilities[i], *vexStatements[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestVEXStatement failed with err: %v", err) - } - } - modelVexStatementIDs = append(modelVexStatementIDs, certVex.ID) - } - return modelVexStatementIDs, nil -} - -func (c *demoClient) IngestVEXStatement(ctx context.Context, subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec) (*model.CertifyVEXStatement, error) { - return c.ingestVEXStatement(ctx, subject, vulnerability, vexStatement, true) -} - -func (c *demoClient) ingestVEXStatement(ctx context.Context, subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec, readOnly bool) (*model.CertifyVEXStatement, error) { - funcName := "IngestVEXStatement" - - lock(&c.m, readOnly) - defer unlock(&c.m, readOnly) - - var packageID uint32 - var foundPkgVersionNode *pkgVersionNode - var artifactID uint32 - var foundArtStrct *artStruct - var subjectVexLinks []uint32 - if subject.Package != nil { - var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundPkgVersionNode, err = byID[*pkgVersionNode](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectVexLinks = foundPkgVersionNode.vexLinks - } else { - var err error - artifactID, err = getArtifactIDFromInput(c, *subject.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundArtStrct, err = byID[*artStruct](artifactID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectVexLinks = foundArtStrct.vexLinks - } - - var vulnerabilityVexLinks []uint32 - vulnID, err := getVulnerabilityIDFromInput(c, vulnerability) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundVulnNode, err := byID[*vulnIDNode](vulnID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - vulnerabilityVexLinks = foundVulnNode.vexLinks - - var searchIDs []uint32 - if len(subjectVexLinks) < len(vulnerabilityVexLinks) { - searchIDs = subjectVexLinks - } else { - searchIDs = vulnerabilityVexLinks - } - - // Don't insert duplicates - duplicate := false - collectedCertifyVexLink := vexLink{} - for _, id := range searchIDs { - v, err := byID[*vexLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - vulnMatch := false - subjectMatch := false - if vulnID != 0 && vulnID == v.vulnerabilityID { - vulnMatch = true - } - if packageID != 0 && packageID == v.packageID { - subjectMatch = true - } - if artifactID != 0 && artifactID == v.artifactID { - subjectMatch = true - } - if vulnMatch && subjectMatch && vexStatement.KnownSince.Equal(v.knownSince) && vexStatement.VexJustification == v.justification && - vexStatement.Status == v.status && vexStatement.Statement == v.statement && vexStatement.StatusNotes == v.statusNotes && - vexStatement.Origin == v.origin && vexStatement.Collector == v.collector { - - collectedCertifyVexLink = *v - duplicate = true - break - } - } - if !duplicate { - if readOnly { - c.m.RUnlock() - v, err := c.ingestVEXStatement(ctx, subject, vulnerability, vexStatement, false) - c.m.RLock() // relock so that defer unlock does not panic - return v, err - } - // store the link - collectedCertifyVexLink = vexLink{ - id: c.getNextID(), - packageID: packageID, - artifactID: artifactID, - vulnerabilityID: vulnID, - knownSince: vexStatement.KnownSince.UTC(), - status: vexStatement.Status, - justification: vexStatement.VexJustification, - statement: vexStatement.Statement, - statusNotes: vexStatement.StatusNotes, - origin: vexStatement.Origin, - collector: vexStatement.Collector, - } - c.index[collectedCertifyVexLink.id] = &collectedCertifyVexLink - c.vexs = append(c.vexs, &collectedCertifyVexLink) - // set the backlinks - if packageID != 0 { - foundPkgVersionNode.setVexLinks(collectedCertifyVexLink.id) - } - if artifactID != 0 { - foundArtStrct.setVexLinks(collectedCertifyVexLink.id) - } - if vulnID != 0 { - foundVulnNode.setVexLinks(collectedCertifyVexLink.id) - } - } - - // build return GraphQL type - builtCertifyVex, err := c.buildCertifyVEXStatement(&collectedCertifyVexLink, nil, true) - if err != nil { - return nil, err - } - return builtCertifyVex, nil -} - -// Query CertifyVex -func (c *demoClient) CertifyVEXStatement(ctx context.Context, filter *model.CertifyVEXStatementSpec) ([]*model.CertifyVEXStatement, error) { - c.m.RLock() - defer c.m.RUnlock() - funcName := "CertifyVEXStatement" - - if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*vexLink](id, c) - if err != nil { - // Not found - return nil, nil - } - // If found by id, ignore rest of fields in spec and return as a match - foundCertifyVex, err := c.buildCertifyVEXStatement(link, filter, true) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - return []*model.CertifyVEXStatement{foundCertifyVex}, nil - } - - var search []uint32 - foundOne := false - - if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Subject.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if exactArtifact != nil { - search = append(search, exactArtifact.vexLinks...) - foundOne = true - } - } - if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Package != nil { - pkgs, err := c.findPackageVersion(filter.Subject.Package) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - foundOne = len(pkgs) > 0 - for _, pkg := range pkgs { - search = append(search, pkg.vexLinks...) - } - } - if !foundOne && filter != nil && filter.Vulnerability != nil { - exactVuln, err := c.exactVulnerability(filter.Vulnerability) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if exactVuln != nil { - search = append(search, exactVuln.vexLinks...) - foundOne = true - } - } - - var out []*model.CertifyVEXStatement - if foundOne { - for _, id := range search { - link, err := byID[*vexLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - out, err = c.addVexIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } else { - for _, link := range c.vexs { - var err error - out, err = c.addVexIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } - return out, nil -} - -func (c *demoClient) addVexIfMatch(out []*model.CertifyVEXStatement, - filter *model.CertifyVEXStatementSpec, link *vexLink) ( - []*model.CertifyVEXStatement, error) { - - if filter != nil && filter.KnownSince != nil && !filter.KnownSince.Equal(link.knownSince) { - return out, nil - } - if filter != nil && filter.VexJustification != nil && *filter.VexJustification != link.justification { - return out, nil - } - if filter != nil && filter.Status != nil && *filter.Status != link.status { - return out, nil - } - if filter != nil && noMatch(filter.Statement, link.statement) { - return out, nil - } - if filter != nil && noMatch(filter.StatusNotes, link.statusNotes) { - return out, nil - } - if filter != nil && noMatch(filter.Collector, link.collector) { - return out, nil - } - if filter != nil && noMatch(filter.Origin, link.origin) { - return out, nil - } - - foundCertifyVex, err := c.buildCertifyVEXStatement(link, filter, false) - if err != nil { - return nil, err - } - if foundCertifyVex == nil { - return out, nil - } - return append(out, foundCertifyVex), nil - -} - -func (c *demoClient) buildCertifyVEXStatement(link *vexLink, filter *model.CertifyVEXStatementSpec, ingestOrIDProvided bool) (*model.CertifyVEXStatement, error) { - var p *model.Package - var a *model.Artifact - var vuln *model.Vulnerability - var err error - if filter != nil && filter.Subject != nil { - if filter.Subject.Package != nil && link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, filter.Subject.Package) - if err != nil { - return nil, err - } - } - if filter.Subject.Artifact != nil && link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, filter.Subject.Artifact) - if err != nil { - return nil, err - } - } - } else { - if link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, nil) - if err != nil { - return nil, err - } - } - if link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, nil) - if err != nil { - return nil, err - } - } - } - - if filter != nil && filter.Vulnerability != nil { - if filter.Vulnerability != nil && link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, filter.Vulnerability) - if err != nil { - return nil, err - } - } - } else { - if link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, nil) - if err != nil { - return nil, err - } - } - } - - var subj model.PackageOrArtifact - if link.packageID != 0 { - if p == nil && ingestOrIDProvided { - return nil, gqlerror.Errorf("failed to retrieve package via packageID") - } else if p == nil && !ingestOrIDProvided { - return nil, nil - } - subj = p - } - if link.artifactID != 0 { - if a == nil && ingestOrIDProvided { - return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") - } else if a == nil && !ingestOrIDProvided { - return nil, nil - } - subj = a - } - - if link.vulnerabilityID != 0 { - if vuln == nil && ingestOrIDProvided { - return nil, gqlerror.Errorf("failed to retrieve vuln via vulnID") - } else if vuln == nil && !ingestOrIDProvided { - return nil, nil - } - } - - certifyVuln := model.CertifyVEXStatement{ - ID: nodeID(link.id), - Subject: subj, - Vulnerability: vuln, - Status: link.status, - VexJustification: link.justification, - Statement: link.statement, - StatusNotes: link.statusNotes, - KnownSince: link.knownSince, - Origin: link.origin, - Collector: link.collector, - } - return &certifyVuln, nil -} diff --git a/pkg/assembler/backends/inmem/hasSBOM.go b/pkg/assembler/backends/inmem/hasSBOM.go deleted file mode 100644 index 2fd24b0920..0000000000 --- a/pkg/assembler/backends/inmem/hasSBOM.go +++ /dev/null @@ -1,465 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "fmt" - "reflect" - "strconv" - "strings" - "time" - - "github.com/vektah/gqlparser/v2/gqlerror" - "golang.org/x/exp/slices" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -type hasSBOMList []*hasSBOMStruct -type hasSBOMStruct struct { - id uint32 - pkg uint32 - artifact uint32 - uri string - algorithm string - digest string - downloadLocation string - origin string - collector string - knownSince time.Time - includedSoftware []uint32 - includedDependencies []uint32 - includedOccurrences []uint32 -} - -func (n *hasSBOMStruct) ID() uint32 { return n.id } - -func (n *hasSBOMStruct) Neighbors(allowedEdges edgeMap) []uint32 { - out := []uint32{} - if n.pkg != 0 && allowedEdges[model.EdgeHasSbomPackage] { - out = append(out, n.pkg) - } - if n.artifact != 0 && allowedEdges[model.EdgeHasSbomArtifact] { - out = append(out, n.artifact) - } - if allowedEdges[model.EdgeHasSbomIncludedSoftware] { - out = append(out, n.includedSoftware...) - } - if allowedEdges[model.EdgeHasSbomIncludedDependencies] { - out = append(out, n.includedDependencies...) - } - if allowedEdges[model.EdgeHasSbomIncludedOccurrences] { - out = append(out, n.includedOccurrences...) - } - return sortAndRemoveDups(out) -} - -func (n *hasSBOMStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convHasSBOM(n) -} - -// Ingest HasSBOM - -func (c *demoClient) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec, includes []*model.HasSBOMIncludesInputSpec) ([]*model.HasSbom, error) { - var modelHasSboms []*model.HasSbom - - for i := range hasSBOMs { - var hasSBOM *model.HasSbom - var err error - if len(subjects.Packages) > 0 { - subject := model.PackageOrArtifactInput{Package: subjects.Packages[i]} - hasSBOM, err = c.IngestHasSbom(ctx, subject, *hasSBOMs[i], *includes[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestHasSbom failed with err: %v", err) - } - } else { - subject := model.PackageOrArtifactInput{Artifact: subjects.Artifacts[i]} - hasSBOM, err = c.IngestHasSbom(ctx, subject, *hasSBOMs[i], *includes[i]) - if err != nil { - return nil, gqlerror.Errorf("IngestHasSbom failed with err: %v", err) - } - } - modelHasSboms = append(modelHasSboms, hasSBOM) - } - return modelHasSboms, nil -} - -func (c *demoClient) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, input model.HasSBOMInputSpec, includes model.HasSBOMIncludesInputSpec) (*model.HasSbom, error) { - funcName := "IngestHasSbom" - - var softwareIDs []uint32 - var dependencyIDs []uint32 - var occurrenceIDs []uint32 - var err error - - if includes.Software != nil { - if softwareIDs, err = parseIDs(includes.Software); err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - for _, id := range softwareIDs { - if err := c.validateSoftwareId(funcName, id); err != nil { - return nil, err - } - } - softwareIDs = sortAndRemoveDups(softwareIDs) - } - if includes.Dependencies != nil { - if dependencyIDs, err = parseIDs(includes.Dependencies); err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - for _, id := range dependencyIDs { - if _, err := byID[*isDependencyLink](id, c); err != nil { - return nil, gqlerror.Errorf("%v :: dependency id %d is not an ingested isDependency", funcName, id) - } - } - dependencyIDs = sortAndRemoveDups(dependencyIDs) - } - if includes.Occurrences != nil { - if occurrenceIDs, err = parseIDs(includes.Occurrences); err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - for _, id := range occurrenceIDs { - if isOccurrence, err := byID[*isOccurrenceStruct](id, c); isOccurrence == nil || err != nil { - return nil, gqlerror.Errorf("%v :: occurrence id %d is not an ingested isOccurrence", funcName, id) - } - } - occurrenceIDs = sortAndRemoveDups(occurrenceIDs) - } - return c.ingestHasSbom(ctx, subject, input, softwareIDs, dependencyIDs, occurrenceIDs, true) -} - -func (c *demoClient) ingestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, input model.HasSBOMInputSpec, includedSoftware, includedDependencies, includedOccurrences []uint32, readOnly bool) (*model.HasSbom, error) { - funcName := "IngestHasSbom" - lock(&c.m, readOnly) - defer unlock(&c.m, readOnly) - - var search []uint32 - var packageID uint32 - var pkg *pkgVersionNode - var artID uint32 - var art *artStruct - if subject.Package != nil { - pmt := model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion} - var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, pmt) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - pkg, err = byID[*pkgVersionNode](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - search = pkg.hasSBOMs - } else { - var err error - art, err = c.artifactByKey(subject.Artifact.Algorithm, subject.Artifact.Digest) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - artID = art.id - search = art.hasSBOMs - } - - algorithm := strings.ToLower(input.Algorithm) - digest := strings.ToLower(input.Digest) - - for _, id := range search { - h, err := byID[*hasSBOMStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if h.pkg == packageID && - h.artifact == artID && - h.uri == input.URI && - h.algorithm == algorithm && - h.digest == digest && - h.downloadLocation == input.DownloadLocation && - h.origin == input.Origin && - h.collector == input.Collector && - input.KnownSince.Equal(h.knownSince) && - slices.Equal(h.includedSoftware, includedSoftware) && - slices.Equal(h.includedDependencies, includedDependencies) && - slices.Equal(h.includedOccurrences, includedOccurrences) { - return c.convHasSBOM(h) - } - } - - if readOnly { - c.m.RUnlock() - b, err := c.ingestHasSbom(ctx, subject, input, includedSoftware, includedDependencies, includedOccurrences, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - - h := &hasSBOMStruct{ - id: c.getNextID(), - pkg: packageID, - artifact: artID, - uri: input.URI, - algorithm: algorithm, - digest: digest, - downloadLocation: input.DownloadLocation, - origin: input.Origin, - collector: input.Collector, - knownSince: input.KnownSince.UTC(), - includedSoftware: includedSoftware, - includedDependencies: includedDependencies, - includedOccurrences: includedOccurrences, - } - c.index[h.id] = h - c.hasSBOMs = append(c.hasSBOMs, h) - if packageID != 0 { - pkg.setHasSBOM(h.id) - } else { - art.setHasSBOMs(h.id) - } - return c.convHasSBOM(h) -} - -func (c *demoClient) convHasSBOM(in *hasSBOMStruct) (*model.HasSbom, error) { - out := &model.HasSbom{ - ID: nodeID(in.id), - URI: in.uri, - Algorithm: in.algorithm, - Digest: in.digest, - DownloadLocation: in.downloadLocation, - Origin: in.origin, - Collector: in.collector, - KnownSince: in.knownSince.UTC(), - } - if in.pkg != 0 { - p, err := c.buildPackageResponse(in.pkg, nil) - if err != nil { - return nil, err - } - out.Subject = p - } else { - art, err := byID[*artStruct](in.artifact, c) - if err != nil { - return nil, err - } - out.Subject = c.convArtifact(art) - } - if len(in.includedSoftware) > 0 { - out.IncludedSoftware = make([]model.PackageOrArtifact, 0, len(in.includedSoftware)) - for _, id := range in.includedSoftware { - switch node := c.index[id].(type) { - case *pkgVersionNode: - if pkg, err := c.buildPackageResponse(id, nil); err != nil { - return nil, err - } else { - out.IncludedSoftware = append(out.IncludedSoftware, pkg) - } - case *artStruct: - if art, err := c.buildArtifactResponse(id, nil); err != nil { - return nil, err - } else { - out.IncludedSoftware = append(out.IncludedSoftware, art) - } - default: - return nil, fmt.Errorf("expected Package or Artifact, found %s", reflect.TypeOf(node)) - } - } - } - if len(in.includedDependencies) > 0 { - out.IncludedDependencies = make([]*model.IsDependency, 0, len(in.includedDependencies)) - for _, id := range in.includedDependencies { - switch node := c.index[id].(type) { - case *isDependencyLink: - if isDep, err := c.buildIsDependency(node, nil, true); err != nil { - return nil, err - } else { - out.IncludedDependencies = append(out.IncludedDependencies, isDep) - } - default: - return nil, fmt.Errorf("expected IsDependency, found %s", reflect.TypeOf(node)) - } - } - } - if len(in.includedOccurrences) > 0 { - out.IncludedOccurrences = make([]*model.IsOccurrence, 0, len(in.includedOccurrences)) - for _, id := range in.includedOccurrences { - switch node := c.index[id].(type) { - case *isOccurrenceStruct: - if isOcc, err := c.convOccurrence(node); err != nil { - return nil, err - } else { - out.IncludedOccurrences = append(out.IncludedOccurrences, isOcc) - } - default: - return nil, fmt.Errorf("expected IsOccurrence, found %s", reflect.TypeOf(node)) - } - } - } - return out, nil -} - -// Query HasSBOM - -func (c *demoClient) HasSBOM(ctx context.Context, filter *model.HasSBOMSpec) ([]*model.HasSbom, error) { - funcName := "HasSBOM" - c.m.RLock() - defer c.m.RUnlock() - - if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %v", funcName, err) - } - id := uint32(id64) - link, err := byID[*hasSBOMStruct](id, c) - if err != nil { - // Not found - return nil, nil - } - // If found by id, ignore rest of fields in spec and return as a match - sb, err := c.convHasSBOM(link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - return []*model.HasSbom{sb}, nil - } - - var search []uint32 - foundOne := false - if filter != nil && filter.Subject != nil && filter.Subject.Package != nil { - pkgs, err := c.findPackageVersion(filter.Subject.Package) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - foundOne = len(pkgs) > 0 - for _, pkg := range pkgs { - search = append(search, pkg.hasSBOMs...) - } - } - if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArt, err := c.artifactExact(filter.Subject.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if exactArt != nil { - search = exactArt.hasSBOMs - foundOne = true - } - } - - var out []*model.HasSbom - if foundOne { - for _, id := range search { - link, err := byID[*hasSBOMStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - out, err = c.addHasSBOMIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } else { - for _, link := range c.hasSBOMs { - var err error - out, err = c.addHasSBOMIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } - - return out, nil -} - -func (c *demoClient) addHasSBOMIfMatch(out []*model.HasSbom, - filter *model.HasSBOMSpec, link *hasSBOMStruct) ( - []*model.HasSbom, error) { - - if filter != nil { - if noMatch(filter.URI, link.uri) || - noMatch(toLower(filter.Algorithm), link.algorithm) || - noMatch(toLower(filter.Digest), link.digest) || - noMatch(filter.DownloadLocation, link.downloadLocation) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) || - (filter.KnownSince != nil && filter.KnownSince.After(link.knownSince)) { - return out, nil - } - // collect packages and artifacts from included software - pkgs, artifacts, err := c.getPackageVersionAndArtifacts(link.includedSoftware) - if err != nil { - return out, err - } - - pkgFilters, artFilters := getPackageAndArtifactFilters(filter.IncludedSoftware) - - if !c.matchPackages(pkgFilters, pkgs) || !c.matchArtifacts(artFilters, artifacts) || - !c.matchDependencies(filter.IncludedDependencies, link.includedDependencies) || - !c.matchOccurrences(filter.IncludedOccurrences, link.includedOccurrences) { - return out, nil - } - - if filter.Subject != nil { - if filter.Subject.Package != nil { - if link.pkg == 0 { - return out, nil - } - p, err := c.buildPackageResponse(link.pkg, filter.Subject.Package) - if err != nil { - return nil, err - } - if p == nil { - return out, nil - } - } else if filter.Subject.Artifact != nil { - if link.artifact == 0 { - return out, nil - } - if !c.artifactMatch(link.artifact, filter.Subject.Artifact) { - return out, nil - } - } - } - } - sb, err := c.convHasSBOM(link) - if err != nil { - return nil, err - } - return append(out, sb), nil -} - -func getPackageAndArtifactFilters(filters []*model.PackageOrArtifactSpec) (pkgs []*model.PkgSpec, arts []*model.ArtifactSpec) { - for _, pkgOrArtSpec := range filters { - if pkgOrArtSpec.Package != nil { - pkgs = append(pkgs, pkgOrArtSpec.Package) - } else if pkgOrArtSpec.Artifact != nil { - arts = append(arts, pkgOrArtSpec.Artifact) - } - } - return -} - -func (c *demoClient) validateSoftwareId(funcName string, id uint32) error { - node, ok := c.index[id] - if !ok { - return gqlerror.Errorf("%v :: software id %d is invalid", funcName, id) - } - if _, ok = node.(*pkgVersionNode); !ok { - _, ok = node.(*artStruct) - } - if !ok { - return gqlerror.Errorf("%v :: software id %d is neither an ingested Package nor an ingested Artifact", funcName, id) - } - return nil -} diff --git a/pkg/assembler/backends/inmem/isOccurrence.go b/pkg/assembler/backends/inmem/isOccurrence.go deleted file mode 100644 index d6f331d96e..0000000000 --- a/pkg/assembler/backends/inmem/isOccurrence.go +++ /dev/null @@ -1,438 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "strconv" - "strings" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// Internal isOccurrence - -type isOccurrenceList []*isOccurrenceStruct -type isOccurrenceStruct struct { - id uint32 - pkg uint32 - source uint32 - artifact uint32 - justification string - origin string - collector string -} - -func (n *isOccurrenceStruct) ID() uint32 { return n.id } - -func (n *isOccurrenceStruct) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 3) - if n.pkg != 0 && allowedEdges[model.EdgeIsOccurrencePackage] { - out = append(out, n.pkg) - } - if n.source != 0 && allowedEdges[model.EdgeIsOccurrenceSource] { - out = append(out, n.source) - } - if allowedEdges[model.EdgeIsOccurrenceArtifact] { - out = append(out, n.artifact) - } - return out -} - -func (n *isOccurrenceStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convOccurrence(n) -} - -// TODO convert to unit tests -// func registerAllIsOccurrence(client *demoClient) error { -// // pkg:conan/openssl.org/openssl@3.0.3?user=bincrafters&channel=stable -// // "conan", "openssl.org", "openssl", "3.0.3", "", "user=bincrafters", "channel=stable" -// selectedType := "conan" -// selectedNameSpace := "openssl.org" -// selectedName := "openssl" -// selectedVersion := "3.0.3" -// qualifierA := "bincrafters" -// qualifierB := "stable" -// selectedQualifiers := []*model.PackageQualifierSpec{{Key: "user", Value: &qualifierA}, {Key: "channel", Value: &qualifierB}} -// selectedPkgSpec := &model.PkgSpec{Type: &selectedType, Namespace: &selectedNameSpace, Name: &selectedName, Version: &selectedVersion, Qualifiers: selectedQualifiers} -// selectedPackage, err := client.Packages(context.TODO(), selectedPkgSpec) -// if err != nil { -// return err -// } -// _, err = client.registerIsOccurrence(selectedPackage[0], nil, &model.Artifact{Digest: "5a787865sd676dacb0142afa0b83029cd7befd9", Algorithm: "sha1"}, "this artifact is an occurrence of this package", "inmem backend", "inmem backend") -// if err != nil { -// return err -// } -// // "git", "github", "github.com/guacsec/guac", "tag=v0.0.1" -// selectedSourceType := "git" -// selectedSourceNameSpace := "github" -// selectedSourceName := "github.com/guacsec/guac" -// selectedTag := "v0.0.1" -// selectedSourceSpec := &model.SourceSpec{Type: &selectedSourceType, Namespace: &selectedSourceNameSpace, Name: &selectedSourceName, Tag: &selectedTag} -// //selectedSource, err := client.Sources(context.TODO(), selectedSourceSpec) -// _, err = client.Sources(context.TODO(), selectedSourceSpec) -// if err != nil { -// return err -// } -// //_, err = client.registerIsOccurrence(nil, selectedSource[0], client.artifacts[0], "this artifact is an occurrence of this source", "inmem backend", "inmem backend") -// if err != nil { -// return err -// } -// } - -// Ingest IngestOccurrences - -func (c *demoClient) IngestOccurrences(ctx context.Context, subjects model.PackageOrSourceInputs, artifacts []*model.ArtifactInputSpec, occurrences []*model.IsOccurrenceInputSpec) ([]*model.IsOccurrence, error) { - var modelIsOccurrences []*model.IsOccurrence - - for i := range occurrences { - var isOccurrence *model.IsOccurrence - var err error - if len(subjects.Packages) > 0 { - subject := model.PackageOrSourceInput{Package: subjects.Packages[i]} - isOccurrence, err = c.IngestOccurrence(ctx, subject, *artifacts[i], *occurrences[i]) - if err != nil { - return nil, gqlerror.Errorf("ingestOccurrence failed with err: %v", err) - } - } else { - subject := model.PackageOrSourceInput{Source: subjects.Sources[i]} - isOccurrence, err = c.IngestOccurrence(ctx, subject, *artifacts[i], *occurrences[i]) - if err != nil { - return nil, gqlerror.Errorf("ingestOccurrence failed with err: %v", err) - } - } - modelIsOccurrences = append(modelIsOccurrences, isOccurrence) - } - return modelIsOccurrences, nil -} - -// Ingest IsOccurrence - -func (c *demoClient) IngestOccurrence(ctx context.Context, subject model.PackageOrSourceInput, artifact model.ArtifactInputSpec, occurrence model.IsOccurrenceInputSpec) (*model.IsOccurrence, error) { - return c.ingestOccurrence(ctx, subject, artifact, occurrence, true) -} - -func (c *demoClient) ingestOccurrence(ctx context.Context, subject model.PackageOrSourceInput, artifact model.ArtifactInputSpec, occurrence model.IsOccurrenceInputSpec, readOnly bool) (*model.IsOccurrence, error) { - funcName := "IngestOccurrence" - - lock(&c.m, readOnly) - defer unlock(&c.m, readOnly) - - a, err := c.artifactByKey(artifact.Algorithm, artifact.Digest) - if err != nil { - return nil, gqlerror.Errorf("%v :: Artifact not found %s", funcName, err) - } - - var packageID uint32 - if subject.Package != nil { - var pmt model.MatchFlags - pmt.Pkg = model.PkgMatchTypeSpecificVersion - pid, err := getPackageIDFromInput(c, *subject.Package, pmt) - if err != nil { - return nil, gqlerror.Errorf("IngestOccurrence :: %v", err) - } - packageID = pid - } - - var sourceID uint32 - if subject.Source != nil { - sid, err := getSourceIDFromInput(c, *subject.Source) - if err != nil { - return nil, gqlerror.Errorf("IngestOccurrence :: %v", err) - } - sourceID = sid - } - - // could search backedges for pkg/src or artifiact, just do artifact - for _, id := range a.occurrences { - o, err := byID[*isOccurrenceStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if o.pkg == packageID && - o.source == sourceID && - o.artifact == a.id && - o.justification == occurrence.Justification && - o.origin == occurrence.Origin && - o.collector == occurrence.Collector { - return c.convOccurrence(o) - } - } - if readOnly { - c.m.RUnlock() - o, err := c.ingestOccurrence(ctx, subject, artifact, occurrence, false) - c.m.RLock() // relock so that defer unlock does not panic - return o, err - } - o := &isOccurrenceStruct{ - id: c.getNextID(), - pkg: packageID, - source: sourceID, - artifact: a.id, - justification: occurrence.Justification, - origin: occurrence.Origin, - collector: occurrence.Collector, - } - c.index[o.id] = o - a.setOccurrences(o.id) - if packageID != 0 { - p, err := byID[*pkgVersionNode](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - p.setOccurrenceLinks(o.id) - } else { - s, err := byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - s.setOccurrenceLinks(o.id) - } - c.occurrences = append(c.occurrences, o) - - return c.convOccurrence(o) -} - -func (c *demoClient) convOccurrence(in *isOccurrenceStruct) (*model.IsOccurrence, error) { - a, err := byID[*artStruct](in.artifact, c) - if err != nil { - return nil, err - } - o := &model.IsOccurrence{ - ID: nodeID(in.id), - Artifact: c.convArtifact(a), - Justification: in.justification, - Origin: in.origin, - Collector: in.collector, - } - if in.pkg != 0 { - p, err := c.buildPackageResponse(in.pkg, nil) - if err != nil { - return nil, err - } - o.Subject = p - } else { - s, err := c.buildSourceResponse(in.source, nil) - if err != nil { - return nil, err - } - o.Subject = s - } - return o, nil -} - -func (c *demoClient) artifactMatch(aID uint32, artifactSpec *model.ArtifactSpec) bool { - if artifactSpec.Digest == nil && artifactSpec.Algorithm == nil { - return true - } - a, _ := c.artifactExact(artifactSpec) - if a != nil && a.id == aID { - return true - } - m, err := byID[*artStruct](aID, c) - if err != nil { - return false - } - if artifactSpec.Digest != nil && strings.ToLower(*artifactSpec.Digest) == m.digest { - return true - } - if artifactSpec.Algorithm != nil && strings.ToLower(*artifactSpec.Algorithm) == m.algorithm { - return true - } - return false -} - -// Query IsOccurrence - -func (c *demoClient) IsOccurrence(ctx context.Context, filter *model.IsOccurrenceSpec) ([]*model.IsOccurrence, error) { - funcName := "IsOccurrence" - - c.m.RLock() - defer c.m.RUnlock() - - if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*isOccurrenceStruct](id, c) - if err != nil { - // Not found - return nil, nil - } - // If found by id, ignore rest of fields in spec and return as a match - o, err := c.convOccurrence(link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - return []*model.IsOccurrence{o}, nil - } - - var search []uint32 - foundOne := false - if filter != nil && filter.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if exactArtifact != nil { - search = append(search, exactArtifact.occurrences...) - foundOne = true - } - } - if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Package != nil { - pkgs, err := c.findPackageVersion(filter.Subject.Package) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - foundOne = len(pkgs) > 0 - for _, pkg := range pkgs { - search = append(search, pkg.occurrences...) - } - } - if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if exactSource != nil { - search = append(search, exactSource.occurrences...) - foundOne = true - } - } - - var out []*model.IsOccurrence - if foundOne { - for _, id := range search { - link, err := byID[*isOccurrenceStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - out, err = c.addOccIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } else { - for _, link := range c.occurrences { - var err error - out, err = c.addOccIfMatch(out, filter, link) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - } - } - return out, nil -} - -func (c *demoClient) addOccIfMatch(out []*model.IsOccurrence, - filter *model.IsOccurrenceSpec, link *isOccurrenceStruct) ( - []*model.IsOccurrence, error) { - - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) { - return out, nil - } - if filter.Artifact != nil && !c.artifactMatch(link.artifact, filter.Artifact) { - return out, nil - } - if filter.Subject != nil { - if filter.Subject.Package != nil { - if link.pkg == 0 { - return out, nil - } - p, err := c.buildPackageResponse(link.pkg, filter.Subject.Package) - if err != nil { - return nil, err - } - if p == nil { - return out, nil - } - } else if filter.Subject.Source != nil { - if link.source == 0 { - return out, nil - } - s, err := c.buildSourceResponse(link.source, filter.Subject.Source) - if err != nil { - return nil, err - } - if s == nil { - return out, nil - } - } - } - o, err := c.convOccurrence(link) - if err != nil { - return nil, err - } - return append(out, o), nil -} - -func (c *demoClient) matchOccurrences(filters []*model.IsOccurrenceSpec, occLinkIDs []uint32 /*, pkgs []uint32, artifacts []uint32*/) bool { - var occLinks []*isOccurrenceStruct - if len(filters) > 0 { - for _, occLinkID := range occLinkIDs { - link, err := byID[*isOccurrenceStruct](occLinkID, c) - if err != nil { - return false - } - occLinks = append(occLinks, link) - } - - for _, filter := range filters { - if filter == nil { - continue - } - if filter.ID != nil { - // Check by ID if present - if !c.isIDPresent(*filter.ID, occLinkIDs) { - return false - } - } else { - // Otherwise match spec information - match := false - for _, link := range occLinks { - if !noMatch(filter.Justification, link.justification) && - !noMatch(filter.Origin, link.origin) && - !noMatch(filter.Collector, link.collector) && - c.matchArtifacts([]*model.ArtifactSpec{filter.Artifact}, []uint32{link.artifact}) { - - if filter.Subject != nil { - if filter.Subject.Package != nil && !c.matchPackages([]*model.PkgSpec{filter.Subject.Package}, []uint32{link.pkg}) { - continue - } else if filter.Subject.Source != nil { - src, err := c.exactSource(filter.Subject.Source) - if err != nil || src == nil { - continue - } - } - } - match = true - break - } - } - if !match { - return false - } - } - } - } - return true -} diff --git a/pkg/assembler/backends/inmem/license.go b/pkg/assembler/backends/inmem/license.go deleted file mode 100644 index fa597677c8..0000000000 --- a/pkg/assembler/backends/inmem/license.go +++ /dev/null @@ -1,183 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// Internal data: Licenses -type licMap map[string]*licStruct -type licStruct struct { - id uint32 - name string - inline string - listVersion string - certifyLegals []uint32 -} - -func (n *licStruct) ID() uint32 { return n.id } - -func (n *licStruct) Neighbors(allowedEdges edgeMap) []uint32 { - if allowedEdges[model.EdgeLicenseCertifyLegal] { - return n.certifyLegals - } - return nil -} - -func (n *licStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convLicense(n), nil -} - -func (n *licStruct) setCertifyLegals(id uint32) { n.certifyLegals = append(n.certifyLegals, id) } - -// Ingest Licenses - -func (c *demoClient) IngestLicenses(ctx context.Context, licenses []*model.LicenseInputSpec) ([]*model.License, error) { - var modelLicenses []*model.License - for _, lic := range licenses { - modelLic, err := c.IngestLicense(ctx, lic) - if err != nil { - return nil, gqlerror.Errorf("ingestLicense failed with err: %v", err) - } - modelLicenses = append(modelLicenses, modelLic) - } - return modelLicenses, nil -} - -func (c *demoClient) IngestLicense(ctx context.Context, license *model.LicenseInputSpec) (*model.License, error) { - return c.ingestLicense(ctx, license, true) -} - -func (c *demoClient) ingestLicense(ctx context.Context, license *model.LicenseInputSpec, readOnly bool) (*model.License, error) { - lock(&c.m, readOnly) - defer unlock(&c.m, readOnly) - - a, ok := c.licenses[licenseKey(license.Name, license.ListVersion)] - if !ok { - if readOnly { - c.m.RUnlock() - a, err := c.ingestLicense(ctx, license, false) - c.m.RLock() // relock so that defer unlock does not panic - return a, err - } - a = &licStruct{ - id: c.getNextID(), - name: license.Name, - } - if license.Inline != nil { - a.inline = *license.Inline - } - if license.ListVersion != nil { - a.listVersion = *license.ListVersion - } - c.index[a.id] = a - c.licenses[licenseKey(license.Name, license.ListVersion)] = a - } - - return c.convLicense(a), nil -} - -func licenseKey(name string, listVersion *string) string { - key := name - if !strings.HasPrefix(name, "LicenseRef") { - key = strings.Join([]string{name, *listVersion}, ":") - } - return key -} - -func (c *demoClient) licenseExact(licenseSpec *model.LicenseSpec) (*licStruct, error) { - - // If ID is provided, try to look up - if licenseSpec.ID != nil { - id64, err := strconv.ParseUint(*licenseSpec.ID, 10, 32) - if err != nil { - return nil, fmt.Errorf("couldn't parse id %w", err) - } - id := uint32(id64) - a, err := byID[*licStruct](id, c) - if err != nil { - // Not found - return nil, nil - } - // If found by id, ignore rest of fields in spec and return as a match - return a, nil - } - - if licenseSpec.Name != nil && strings.HasPrefix(*licenseSpec.Name, "LicenseRef") { - if l, ok := c.licenses[licenseKey(*licenseSpec.Name, nil)]; ok { - if licenseSpec.Inline == nil || - (licenseSpec.Inline != nil && *licenseSpec.Inline == l.inline) { - return l, nil - } - } - } - if licenseSpec.Name != nil && - !strings.HasPrefix(*licenseSpec.Name, "LicenseRef") && - licenseSpec.ListVersion != nil && - licenseSpec.Inline == nil { - if l, ok := c.licenses[licenseKey(*licenseSpec.Name, licenseSpec.ListVersion)]; ok { - return l, nil - } - } - return nil, nil -} - -// Query Licenses - -func (c *demoClient) Licenses(ctx context.Context, licenseSpec *model.LicenseSpec) ([]*model.License, error) { - c.m.RLock() - defer c.m.RUnlock() - a, err := c.licenseExact(licenseSpec) - if err != nil { - return nil, gqlerror.Errorf("Licenses :: invalid spec %s", err) - } - if a != nil { - return []*model.License{c.convLicense(a)}, nil - } - - var rv []*model.License - for _, l := range c.licenses { - if noMatch(licenseSpec.Name, l.name) || - noMatch(licenseSpec.ListVersion, l.listVersion) || - noMatch(licenseSpec.Inline, l.inline) { - continue - } - rv = append(rv, c.convLicense(l)) - } - return rv, nil -} - -func (c *demoClient) convLicense(a *licStruct) *model.License { - rv := &model.License{ - ID: nodeID(a.id), - Name: a.name, - } - if a.inline != "" { - rv.Inline = &a.inline - } - if a.listVersion != "" { - rv.ListVersion = &a.listVersion - } - return rv -} diff --git a/pkg/assembler/backends/inmem/pkg.go b/pkg/assembler/backends/inmem/pkg.go deleted file mode 100644 index b862a31e1e..0000000000 --- a/pkg/assembler/backends/inmem/pkg.go +++ /dev/null @@ -1,918 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "crypto/sha256" - "errors" - "fmt" - "reflect" - "slices" - "strconv" - "strings" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// TODO: move this into a unit test for this file -// func registerAllPackages(client *demoClient) { -// ctx := context.Background() - -// v11 := "2.11.1" -// v12 := "2.12.0" -// subpath1 := "saved_model_cli.py" -// subpath2 := "__init__.py" -// opensslNamespace := "openssl.org" -// opensslVersion := "3.0.3" - -// inputs := []model.PkgInputSpec{{ -// Type: "pypi", -// Name: "tensorflow", -// }, { -// Type: "pypi", -// Name: "tensorflow", -// Version: &v11, -// }, { -// Type: "pypi", -// Name: "tensorflow", -// Version: &v12, -// }, { -// Type: "pypi", -// Name: "tensorflow", -// Version: &v12, -// Subpath: &subpath1, -// }, { -// Type: "pypi", -// Name: "tensorflow", -// Version: &v12, -// Subpath: &subpath2, -// }, { -// Type: "pypi", -// Name: "tensorflow", -// Version: &v12, -// Subpath: &subpath1, -// }, { -// Type: "conan", -// Namespace: &opensslNamespace, -// Name: "openssl", -// Version: &opensslVersion, -// }} - -// for _, input := range inputs { -// _, err := client.IngestPackage(ctx, input) -// if err != nil { -// log.Printf("Error in ingesting: %v\n", err) -// } -// } -// } - -// Internal data: Packages -type pkgTypeMap map[string]*pkgNamespaceStruct -type pkgNamespaceStruct struct { - id uint32 - typeKey string - namespaces pkgNamespaceMap -} -type pkgNamespaceMap map[string]*pkgNameStruct -type pkgNameStruct struct { - id uint32 - parent uint32 - namespace string - names pkgNameMap -} -type pkgNameMap map[string]*pkgVersionStruct -type pkgVersionStruct struct { - id uint32 - parent uint32 - name string - versions pkgVersionMap - srcMapLinks []uint32 - isDependencyLinks []uint32 - badLinks []uint32 - goodLinks []uint32 - hasMetadataLinks []uint32 - pointOfContactLinks []uint32 -} -type pkgVersionNodeHash string -type pkgVersionMap map[pkgVersionNodeHash]*pkgVersionNode -type pkgVersionNode struct { - id uint32 - parent uint32 - version string - subpath string - qualifiers map[string]string - srcMapLinks []uint32 - isDependencyLinks []uint32 - occurrences []uint32 - certifyVulnLinks []uint32 - hasSBOMs []uint32 - vexLinks []uint32 - badLinks []uint32 - goodLinks []uint32 - hasMetadataLinks []uint32 - pointOfContactLinks []uint32 - pkgEquals []uint32 - certifyLegals []uint32 -} - -// Be type safe, don't use any / interface{} -type pkgNameOrVersion interface { - implementsPkgNameOrVersion() - setSrcMapLinks(id uint32) - getSrcMapLinks() []uint32 - setIsDependencyLinks(id uint32) - getIsDependencyLinks() []uint32 - setCertifyBadLinks(id uint32) - getCertifyBadLinks() []uint32 - setCertifyGoodLinks(id uint32) - getCertifyGoodLinks() []uint32 - setHasMetadataLinks(id uint32) - getHasMetadataLinks() []uint32 - setPointOfContactLinks(id uint32) - getPointOfContactLinks() []uint32 - - node -} - -func (n *pkgNamespaceStruct) ID() uint32 { return n.id } -func (n *pkgNameStruct) ID() uint32 { return n.id } -func (n *pkgVersionStruct) ID() uint32 { return n.id } -func (n *pkgVersionNode) ID() uint32 { return n.id } - -func (n *pkgNamespaceStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgePackageTypePackageNamespace] { - for _, v := range n.namespaces { - out = append(out, v.id) - } - } - return out -} -func (n *pkgNameStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgePackageNamespacePackageName] { - for _, v := range n.names { - out = append(out, v.id) - } - } - if allowedEdges[model.EdgePackageNamespacePackageType] { - out = append(out, n.parent) - } - return out -} -func (n *pkgVersionStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgePackageNamePackageNamespace] { - out = append(out, n.parent) - } - if allowedEdges[model.EdgePackageNamePackageVersion] { - for _, v := range n.versions { - out = append(out, v.id) - } - } - if allowedEdges[model.EdgePackageHasSourceAt] { - out = append(out, n.srcMapLinks...) - } - if allowedEdges[model.EdgePackageIsDependency] { - out = append(out, n.isDependencyLinks...) - } - if allowedEdges[model.EdgePackageCertifyBad] { - out = append(out, n.badLinks...) - } - if allowedEdges[model.EdgePackageCertifyGood] { - out = append(out, n.goodLinks...) - } - if allowedEdges[model.EdgePackageHasMetadata] { - out = append(out, n.hasMetadataLinks...) - } - if allowedEdges[model.EdgePackagePointOfContact] { - out = append(out, n.pointOfContactLinks...) - } - - return out -} -func (n *pkgVersionNode) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgePackageVersionPackageName] { - out = append(out, n.parent) - } - if allowedEdges[model.EdgePackageHasSourceAt] { - out = append(out, n.srcMapLinks...) - } - if allowedEdges[model.EdgePackageIsDependency] { - out = append(out, n.isDependencyLinks...) - } - if allowedEdges[model.EdgePackageIsOccurrence] { - out = append(out, n.occurrences...) - } - if allowedEdges[model.EdgePackageCertifyVuln] { - out = append(out, n.certifyVulnLinks...) - } - if allowedEdges[model.EdgePackageHasSbom] { - out = append(out, n.hasSBOMs...) - } - if allowedEdges[model.EdgePackageCertifyVexStatement] { - out = append(out, n.vexLinks...) - } - if allowedEdges[model.EdgePackageCertifyBad] { - out = append(out, n.badLinks...) - } - if allowedEdges[model.EdgePackageCertifyGood] { - out = append(out, n.goodLinks...) - } - if allowedEdges[model.EdgePackagePkgEqual] { - out = append(out, n.pkgEquals...) - } - if allowedEdges[model.EdgePackageHasMetadata] { - out = append(out, n.hasMetadataLinks...) - } - if allowedEdges[model.EdgePackagePointOfContact] { - out = append(out, n.pointOfContactLinks...) - } - if allowedEdges[model.EdgePackageCertifyLegal] { - out = append(out, n.certifyLegals...) - } - - return out -} - -func (n *pkgNamespaceStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildPackageResponse(n.id, nil) -} -func (n *pkgNameStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildPackageResponse(n.id, nil) -} -func (n *pkgVersionStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildPackageResponse(n.id, nil) -} -func (n *pkgVersionNode) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildPackageResponse(n.id, nil) -} - -func (p *pkgVersionStruct) implementsPkgNameOrVersion() {} -func (p *pkgVersionNode) implementsPkgNameOrVersion() {} - -// hasSourceAt back edges -func (p *pkgVersionStruct) setSrcMapLinks(id uint32) { p.srcMapLinks = append(p.srcMapLinks, id) } -func (p *pkgVersionNode) setSrcMapLinks(id uint32) { p.srcMapLinks = append(p.srcMapLinks, id) } -func (p *pkgVersionStruct) getSrcMapLinks() []uint32 { return p.srcMapLinks } -func (p *pkgVersionNode) getSrcMapLinks() []uint32 { return p.srcMapLinks } - -// isDependency back edges -func (p *pkgVersionStruct) setIsDependencyLinks(id uint32) { - p.isDependencyLinks = append(p.isDependencyLinks, id) -} -func (p *pkgVersionNode) setIsDependencyLinks(id uint32) { - p.isDependencyLinks = append(p.isDependencyLinks, id) -} -func (p *pkgVersionStruct) getIsDependencyLinks() []uint32 { return p.isDependencyLinks } -func (p *pkgVersionNode) getIsDependencyLinks() []uint32 { return p.isDependencyLinks } - -// isOccurrence back edges -func (p *pkgVersionNode) setOccurrenceLinks(id uint32) { p.occurrences = append(p.occurrences, id) } - -// certifyVulnerability back edges -func (p *pkgVersionNode) setVulnerabilityLinks(id uint32) { - p.certifyVulnLinks = append(p.certifyVulnLinks, id) -} - -// certifyVexStatement back edges -func (p *pkgVersionNode) setVexLinks(id uint32) { - p.vexLinks = append(p.vexLinks, id) -} - -// hasSBOM back edges -func (p *pkgVersionNode) setHasSBOM(id uint32) { p.hasSBOMs = append(p.hasSBOMs, id) } - -// certifyBad back edges -func (p *pkgVersionStruct) setCertifyBadLinks(id uint32) { p.badLinks = append(p.badLinks, id) } -func (p *pkgVersionNode) setCertifyBadLinks(id uint32) { p.badLinks = append(p.badLinks, id) } -func (p *pkgVersionStruct) getCertifyBadLinks() []uint32 { return p.badLinks } -func (p *pkgVersionNode) getCertifyBadLinks() []uint32 { return p.badLinks } - -// certifyGood back edges -func (p *pkgVersionStruct) setCertifyGoodLinks(id uint32) { p.goodLinks = append(p.goodLinks, id) } -func (p *pkgVersionNode) setCertifyGoodLinks(id uint32) { p.goodLinks = append(p.goodLinks, id) } -func (p *pkgVersionStruct) getCertifyGoodLinks() []uint32 { return p.goodLinks } -func (p *pkgVersionNode) getCertifyGoodLinks() []uint32 { return p.goodLinks } - -// hasMetadata back edges -func (p *pkgVersionStruct) setHasMetadataLinks(id uint32) { - p.hasMetadataLinks = append(p.hasMetadataLinks, id) -} -func (p *pkgVersionNode) setHasMetadataLinks(id uint32) { - p.hasMetadataLinks = append(p.hasMetadataLinks, id) -} -func (p *pkgVersionStruct) getHasMetadataLinks() []uint32 { return p.hasMetadataLinks } -func (p *pkgVersionNode) getHasMetadataLinks() []uint32 { return p.hasMetadataLinks } - -// pointOfContact back edges -func (p *pkgVersionStruct) setPointOfContactLinks(id uint32) { - p.pointOfContactLinks = append(p.pointOfContactLinks, id) -} -func (p *pkgVersionNode) setPointOfContactLinks(id uint32) { - p.pointOfContactLinks = append(p.pointOfContactLinks, id) -} -func (p *pkgVersionStruct) getPointOfContactLinks() []uint32 { return p.pointOfContactLinks } -func (p *pkgVersionNode) getPointOfContactLinks() []uint32 { return p.pointOfContactLinks } - -// pkgEqual back edges -func (p *pkgVersionNode) setPkgEquals(id uint32) { p.pkgEquals = append(p.pkgEquals, id) } - -func (p *pkgVersionNode) setCertifyLegals(id uint32) { p.certifyLegals = append(p.certifyLegals, id) } - -// Ingest Package - -func (c *demoClient) IngestPackages(ctx context.Context, pkgs []*model.PkgInputSpec) ([]*model.Package, error) { - var modelPkgs []*model.Package - for _, pkg := range pkgs { - modelPkg, err := c.IngestPackage(ctx, *pkg) - if err != nil { - return nil, gqlerror.Errorf("ingestPackage failed with err: %v", err) - } - modelPkgs = append(modelPkgs, modelPkg) - } - return modelPkgs, nil -} - -func (c *demoClient) IngestPackage(ctx context.Context, input model.PkgInputSpec) (*model.Package, error) { - c.m.RLock() - namespacesStruct, hasNamespace := c.packages[input.Type] - c.m.RUnlock() - if !hasNamespace { - c.m.Lock() - namespacesStruct, hasNamespace = c.packages[input.Type] - if !hasNamespace { - namespacesStruct = &pkgNamespaceStruct{ - id: c.getNextID(), - typeKey: input.Type, - namespaces: pkgNamespaceMap{}, - } - c.index[namespacesStruct.id] = namespacesStruct - c.packages[input.Type] = namespacesStruct - } - c.m.Unlock() - } - namespaces := namespacesStruct.namespaces - - c.m.RLock() - namesStruct, hasName := namespaces[nilToEmpty(input.Namespace)] - c.m.RUnlock() - if !hasName { - c.m.Lock() - namesStruct, hasName = namespaces[nilToEmpty(input.Namespace)] - if !hasName { - namesStruct = &pkgNameStruct{ - id: c.getNextID(), - parent: namespacesStruct.id, - namespace: nilToEmpty(input.Namespace), - names: pkgNameMap{}, - } - c.index[namesStruct.id] = namesStruct - namespaces[nilToEmpty(input.Namespace)] = namesStruct - } - c.m.Unlock() - } - names := namesStruct.names - - c.m.RLock() - versionStruct, hasVersions := names[input.Name] - c.m.RUnlock() - if !hasVersions { - c.m.Lock() - versionStruct, hasVersions = names[input.Name] - if !hasVersions { - versionStruct = &pkgVersionStruct{ - id: c.getNextID(), - parent: namesStruct.id, - name: input.Name, - versions: pkgVersionMap{}, - } - c.index[versionStruct.id] = versionStruct - names[input.Name] = versionStruct - } - c.m.Unlock() - } - versions := versionStruct.versions - - c.m.RLock() - duplicate, collectedVersion := duplicatePkgVer(versions, input) - c.m.RUnlock() - if !duplicate { - c.m.Lock() - duplicate, collectedVersion = duplicatePkgVer(versions, input) - if !duplicate { - collectedVersion = &pkgVersionNode{ - id: c.getNextID(), - parent: versionStruct.id, - version: nilToEmpty(input.Version), - subpath: nilToEmpty(input.Subpath), - qualifiers: getQualifiersFromInput(input.Qualifiers), - } - c.index[collectedVersion.id] = collectedVersion - - versionDigest, err := hashPkgVersionNode(collectedVersion) - if err != nil { - c.m.Unlock() - return nil, err - } - versions[versionDigest] = collectedVersion - } - c.m.Unlock() - } - - // build return GraphQL type - c.m.RLock() - defer c.m.RUnlock() - return c.buildPackageResponse(collectedVersion.id, nil) -} - -// hash the canonical representation of a version. -func hashPkgVersionNode(version *pkgVersionNode) (pkgVersionNodeHash, error) { - if version == nil { - return "", fmt.Errorf("version is nil") - } - return hashVersionHelper(version.version, version.subpath, version.qualifiers), nil -} - -// hash the canonical representation of a version -func hashPkgInputSpecVersion(input model.PkgInputSpec) pkgVersionNodeHash { - qualifiers := getQualifiersFromInput(input.Qualifiers) - return hashVersionHelper(nilToEmpty(input.Version), nilToEmpty(input.Subpath), qualifiers) -} - -func hashVersionHelper(version string, subpath string, qualifiers map[string]string) pkgVersionNodeHash { - // first sort the qualifiers - qualifierSlice := make([]string, 0, len(qualifiers)) - for key, value := range qualifiers { - qualifierSlice = append(qualifierSlice, fmt.Sprintf("%s:%s", key, value)) - } - slices.Sort(qualifierSlice) - qualifiersStr := strings.Join(qualifierSlice, ",") - - canonicalVersion := fmt.Sprintf("%s,%s,%s", version, subpath, qualifiersStr) - digest := sha256.Sum256([]byte(canonicalVersion)) - return pkgVersionNodeHash(fmt.Sprintf("%x", digest)) -} - -func duplicatePkgVer(versions pkgVersionMap, input model.PkgInputSpec) (bool, *pkgVersionNode) { - digest := hashPkgInputSpecVersion(input) - - if version, ok := versions[digest]; ok { - return true, version - } - return false, nil -} - -// Query Package -func (c *demoClient) Packages(ctx context.Context, filter *model.PkgSpec) ([]*model.Package, error) { - c.m.RLock() - defer c.m.RUnlock() - if filter != nil && filter.ID != nil { - id, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - p, err := c.buildPackageResponse(uint32(id), filter) - if err != nil { - if errors.Is(err, errNotFound) { - // not found - return nil, nil - } - return nil, err - } - return []*model.Package{p}, nil - } - out := []*model.Package{} - - if filter != nil && filter.Type != nil { - pkgNamespaceStruct, ok := c.packages[*filter.Type] - if ok { - pNamespaces := buildPkgNamespace(pkgNamespaceStruct, filter) - if len(pNamespaces) > 0 { - out = append(out, &model.Package{ - ID: nodeID(pkgNamespaceStruct.id), - Type: pkgNamespaceStruct.typeKey, - Namespaces: pNamespaces, - }) - } - } - } else { - for dbType, pkgNamespaceStruct := range c.packages { - pNamespaces := buildPkgNamespace(pkgNamespaceStruct, filter) - if len(pNamespaces) > 0 { - out = append(out, &model.Package{ - ID: nodeID(pkgNamespaceStruct.id), - Type: dbType, - Namespaces: pNamespaces, - }) - } - } - } - return out, nil -} - -func buildPkgNamespace(pkgNamespaceStruct *pkgNamespaceStruct, filter *model.PkgSpec) []*model.PackageNamespace { - pNamespaces := []*model.PackageNamespace{} - if filter != nil && filter.Namespace != nil { - pkgNameStruct, ok := pkgNamespaceStruct.namespaces[*filter.Namespace] - if ok { - pns := buildPkgName(pkgNameStruct, filter) - if len(pns) > 0 { - pNamespaces = append(pNamespaces, &model.PackageNamespace{ - ID: nodeID(pkgNameStruct.id), - Namespace: pkgNameStruct.namespace, - Names: pns, - }) - } - } - } else { - for namespace, pkgNameStruct := range pkgNamespaceStruct.namespaces { - pns := buildPkgName(pkgNameStruct, filter) - if len(pns) > 0 { - pNamespaces = append(pNamespaces, &model.PackageNamespace{ - ID: nodeID(pkgNameStruct.id), - Namespace: namespace, - Names: pns, - }) - } - } - } - return pNamespaces -} - -func buildPkgName(pkgNameStruct *pkgNameStruct, filter *model.PkgSpec) []*model.PackageName { - pns := []*model.PackageName{} - if filter != nil && filter.Name != nil { - pkgVersionStruct, ok := pkgNameStruct.names[*filter.Name] - if ok { - pvs := buildPkgVersion(pkgVersionStruct, filter) - if len(pvs) > 0 { - pns = append(pns, &model.PackageName{ - ID: nodeID(pkgVersionStruct.id), - Name: pkgVersionStruct.name, - Versions: pvs, - }) - } - } - } else { - for name, pkgVersionStruct := range pkgNameStruct.names { - pvs := buildPkgVersion(pkgVersionStruct, filter) - if len(pvs) > 0 { - pns = append(pns, &model.PackageName{ - ID: nodeID(pkgVersionStruct.id), - Name: name, - Versions: pvs, - }) - } - } - } - return pns -} - -func buildPkgVersion(pkgVersionStruct *pkgVersionStruct, filter *model.PkgSpec) []*model.PackageVersion { - pvs := []*model.PackageVersion{} - for _, v := range pkgVersionStruct.versions { - if filter != nil && noMatch(filter.Version, v.version) { - continue - } - if filter != nil && noMatch(filter.Subpath, v.subpath) { - continue - } - if filter != nil && noMatchQualifiers(filter, v.qualifiers) { - continue - } - pvs = append(pvs, &model.PackageVersion{ - ID: nodeID(v.id), - Version: v.version, - Subpath: v.subpath, - Qualifiers: getCollectedPackageQualifiers(v.qualifiers), - }) - } - return pvs -} - -// Builds a model.Package to send as GraphQL response, starting from id. -// The optional filter allows restricting output (on selection operations). -func (c *demoClient) buildPackageResponse(id uint32, filter *model.PkgSpec) (*model.Package, error) { - if filter != nil && filter.ID != nil { - filteredID, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - if uint32(filteredID) != id { - return nil, nil - } - } - - node, ok := c.index[id] - if !ok { - return nil, fmt.Errorf("%w : ID does not match existing node", errNotFound) - } - - pvl := []*model.PackageVersion{} - if versionNode, ok := node.(*pkgVersionNode); ok { - if filter != nil && noMatch(filter.Version, versionNode.version) { - return nil, nil - } - if filter != nil && noMatch(filter.Subpath, versionNode.subpath) { - return nil, nil - } - if filter != nil && noMatchQualifiers(filter, versionNode.qualifiers) { - return nil, nil - } - pvl = append(pvl, &model.PackageVersion{ - ID: nodeID(versionNode.id), - Version: versionNode.version, - Subpath: versionNode.subpath, - Qualifiers: getCollectedPackageQualifiers(versionNode.qualifiers), - }) - node, ok = c.index[versionNode.parent] - if !ok { - return nil, fmt.Errorf("Internal ID does not match existing node") - } - } - - pnl := []*model.PackageName{} - if versionStruct, ok := node.(*pkgVersionStruct); ok { - if filter != nil && noMatch(filter.Name, versionStruct.name) { - return nil, nil - } - pnl = append(pnl, &model.PackageName{ - ID: nodeID(versionStruct.id), - Name: versionStruct.name, - Versions: pvl, - }) - node, ok = c.index[versionStruct.parent] - if !ok { - return nil, fmt.Errorf("Internal ID does not match existing node") - } - } - - pnsl := []*model.PackageNamespace{} - if nameStruct, ok := node.(*pkgNameStruct); ok { - if filter != nil && noMatch(filter.Namespace, nameStruct.namespace) { - return nil, nil - } - pnsl = append(pnsl, &model.PackageNamespace{ - ID: nodeID(nameStruct.id), - Namespace: nameStruct.namespace, - Names: pnl, - }) - node, ok = c.index[nameStruct.parent] - if !ok { - return nil, fmt.Errorf("Internal ID does not match existing node") - } - } - - namespaceStruct, ok := node.(*pkgNamespaceStruct) - if !ok { - return nil, fmt.Errorf("%w: ID does not match expected node type for package namespace", errNotFound) - } - p := model.Package{ - ID: nodeID(namespaceStruct.id), - Type: namespaceStruct.typeKey, - Namespaces: pnsl, - } - if filter != nil && noMatch(filter.Type, p.Type) { - return nil, nil - } - return &p, nil -} - -func getPackageIDFromInput(c *demoClient, input model.PkgInputSpec, pkgMatchType model.MatchFlags) (uint32, error) { - pkgNamespace, pkgHasNamespace := c.packages[input.Type] - if !pkgHasNamespace { - return 0, gqlerror.Errorf("Package type \"%s\" not found", input.Type) - } - pkgName, pkgHasName := pkgNamespace.namespaces[nilToEmpty(input.Namespace)] - if !pkgHasName { - return 0, gqlerror.Errorf("Package namespace \"%s\" not found", nilToEmpty(input.Namespace)) - } - pkgVersion, pkgHasVersion := pkgName.names[input.Name] - if !pkgHasVersion { - return 0, gqlerror.Errorf("Package name \"%s\" not found", input.Name) - } - var packageID uint32 - if pkgMatchType.Pkg == model.PkgMatchTypeAllVersions { - packageID = pkgVersion.id - } else { - found := false - for _, version := range pkgVersion.versions { - if noMatchInput(input.Version, version.version) { - continue - } - if noMatchInput(input.Subpath, version.subpath) { - continue - } - if !reflect.DeepEqual(version.qualifiers, getQualifiersFromInput(input.Qualifiers)) { - continue - } - if found { - return 0, gqlerror.Errorf("More than one package matches input") - } - packageID = version.id - found = true - } - if !found { - return 0, gqlerror.Errorf("No package matches input") - } - } - return packageID, nil -} - -func (c *demoClient) matchPackages(filter []*model.PkgSpec, pkgs []uint32) bool { - pkgs = slices.Clone(pkgs) - pkgs = sortAndRemoveDups(pkgs) - - for _, pvSpec := range filter { - if pvSpec != nil { - if pvSpec.ID != nil { - // Check by ID if present - if !c.isIDPresent(*pvSpec.ID, pkgs) { - return false - } - } else { - // Otherwise match spec information - match := false - for _, pkgId := range pkgs { - id := pkgId - pkgVersion, err := byID[*pkgVersionNode](id, c) - if err == nil { - if noMatch(pvSpec.Subpath, pkgVersion.subpath) || noMatchQualifiers(pvSpec, pkgVersion.qualifiers) || noMatch(pvSpec.Version, pkgVersion.version) { - continue - } - id = pkgVersion.parent - } - pkgName, err := byID[*pkgVersionStruct](id, c) - if err == nil { - if noMatch(pvSpec.Name, pkgName.name) { - continue - } - id = pkgName.parent - } - pkgNamespace, err := byID[*pkgNameStruct](id, c) - if err == nil { - if noMatch(pvSpec.Namespace, pkgNamespace.namespace) { - continue - } - id = pkgNamespace.parent - } - pkgType, err := byID[*pkgNamespaceStruct](id, c) - if err == nil { - if noMatch(pvSpec.Type, pkgType.typeKey) { - continue - } else { - match = true - break - } - } - } - if !match { - return false - } - } - } - } - return true -} - -func getCollectedPackageQualifiers(qualifierMap map[string]string) []*model.PackageQualifier { - qualifiers := []*model.PackageQualifier{} - for key, val := range qualifierMap { - qualifier := &model.PackageQualifier{ - Key: key, - Value: val, - } - qualifiers = append(qualifiers, qualifier) - - } - return qualifiers -} - -func getQualifiersFromInput(qualifiersSpec []*model.PackageQualifierInputSpec) map[string]string { - qualifiersMap := map[string]string{} - if qualifiersSpec == nil { - return qualifiersMap - } - for _, kv := range qualifiersSpec { - qualifiersMap[kv.Key] = kv.Value - } - return qualifiersMap -} - -func getQualifiersFromFilter(qualifiersSpec []*model.PackageQualifierSpec) map[string]string { - qualifiersMap := map[string]string{} - if qualifiersSpec == nil { - return qualifiersMap - } - for _, kv := range qualifiersSpec { - qualifiersMap[kv.Key] = nilToEmpty(kv.Value) - } - return qualifiersMap -} - -func noMatchQualifiers(filter *model.PkgSpec, v map[string]string) bool { - // Allow matching on nodes with no qualifiers - if filter.MatchOnlyEmptyQualifiers != nil { - if *filter.MatchOnlyEmptyQualifiers && len(v) != 0 { - return true - } - } - if filter.Qualifiers != nil && len(filter.Qualifiers) > 0 { - filterQualifiers := getQualifiersFromFilter(filter.Qualifiers) - return !reflect.DeepEqual(v, filterQualifiers) - } - return false -} - -func (c *demoClient) findPackageVersion(filter *model.PkgSpec) ([]*pkgVersionNode, error) { - if filter == nil { - return nil, nil - } - if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - id := uint32(id64) - if node, ok := c.index[id]; ok { - if c, ok := node.(*pkgVersionNode); ok { - return []*pkgVersionNode{c}, nil - } - } - } - out := make([]*pkgVersionNode, 0) - if filter.Type != nil && filter.Namespace != nil && filter.Name != nil && filter.Version != nil { - tp, ok := c.packages[*filter.Type] - if !ok { - return nil, nil - } - ns, ok := tp.namespaces[*filter.Namespace] - if !ok { - return nil, nil - } - nm, ok := ns.names[*filter.Name] - if !ok { - return nil, nil - } - for _, v := range nm.versions { - if *filter.Version != v.version || - noMatch(filter.Subpath, v.subpath) || - noMatchQualifiers(filter, v.qualifiers) { - continue - } - out = append(out, v) - } - } - return out, nil -} - -func (c *demoClient) exactPackageName(filter *model.PkgSpec) (*pkgVersionStruct, error) { - if filter == nil { - return nil, nil - } - if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - id := uint32(id64) - if node, ok := c.index[id]; ok { - if c, ok := node.(*pkgVersionStruct); ok { - return c, nil - } - } - } - if filter.Type != nil && filter.Namespace != nil && filter.Name != nil { - tp, ok := c.packages[*filter.Type] - if !ok { - return nil, nil - } - ns, ok := tp.namespaces[*filter.Namespace] - if !ok { - return nil, nil - } - nm, ok := ns.names[*filter.Name] - if !ok { - return nm, nil - } - } - return nil, nil -} diff --git a/pkg/assembler/backends/inmem/src.go b/pkg/assembler/backends/inmem/src.go deleted file mode 100644 index 67f8836933..0000000000 --- a/pkg/assembler/backends/inmem/src.go +++ /dev/null @@ -1,516 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "errors" - "fmt" - "strconv" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -// TODO: move this into a unit test for this file -// func registerAllSources(client *demoClient) { -// ctx := context.Background() -// v12 := "v2.12.0" -// commit := "abcdef" - -// inputs := []model.SourceInputSpec{{ -// Type: "git", -// Namespace: "github.com", -// Name: "tensorflow", -// }, { -// Type: "git", -// Namespace: "github.com", -// Name: "build", -// }, { -// Type: "git", -// Namespace: "github.com", -// Name: "tensorflow", -// Tag: &v12, -// }, { -// Type: "git", -// Namespace: "github.com", -// Name: "tensorflow", -// Commit: &commit, -// }} - -// for _, input := range inputs { -// _, err := client.IngestSource(ctx, input) -// if err != nil { -// log.Printf("Error in ingesting: %v\n", err) -// } -// } -// } - -// Internal data: Sources -type srcTypeMap map[string]*srcNamespaceStruct -type srcNamespaceStruct struct { - id uint32 - typeKey string - namespaces srcNamespaceMap -} -type srcNamespaceMap map[string]*srcNameStruct -type srcNameStruct struct { - id uint32 - parent uint32 - namespace string - names srcNameList -} -type srcNameList []*srcNameNode -type srcNameNode struct { - id uint32 - parent uint32 - name string - tag string - commit string - srcMapLinks []uint32 - scorecardLinks []uint32 - occurrences []uint32 - badLinks []uint32 - goodLinks []uint32 - hasMetadataLinks []uint32 - pointOfContactLinks []uint32 - certifyLegals []uint32 -} - -func (n *srcNamespaceStruct) ID() uint32 { return n.id } -func (n *srcNameStruct) ID() uint32 { return n.id } -func (n *srcNameNode) ID() uint32 { return n.id } - -func (n *srcNamespaceStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgeSourceTypeSourceNamespace] { - for _, v := range n.namespaces { - out = append(out, v.id) - } - } - return out -} -func (n *srcNameStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgeSourceNamespaceSourceName] { - for _, v := range n.names { - out = append(out, v.id) - } - } - if allowedEdges[model.EdgeSourceNamespaceSourceType] { - out = append(out, n.parent) - } - return out -} -func (n *srcNameNode) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgeSourceNameSourceNamespace] { - out = append(out, n.parent) - } - if allowedEdges[model.EdgeSourceHasSourceAt] { - out = append(out, n.srcMapLinks...) - } - if allowedEdges[model.EdgeSourceCertifyScorecard] { - out = append(out, n.scorecardLinks...) - } - if allowedEdges[model.EdgeSourceIsOccurrence] { - out = append(out, n.occurrences...) - } - if allowedEdges[model.EdgeSourceCertifyBad] { - out = append(out, n.badLinks...) - } - if allowedEdges[model.EdgeSourceCertifyGood] { - out = append(out, n.goodLinks...) - } - if allowedEdges[model.EdgeSourceHasMetadata] { - out = append(out, n.hasMetadataLinks...) - } - if allowedEdges[model.EdgeSourcePointOfContact] { - out = append(out, n.pointOfContactLinks...) - } - if allowedEdges[model.EdgeSourceCertifyLegal] { - out = append(out, n.certifyLegals...) - } - - return out -} - -func (n *srcNamespaceStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildSourceResponse(n.id, nil) -} -func (n *srcNameStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildSourceResponse(n.id, nil) -} -func (n *srcNameNode) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildSourceResponse(n.id, nil) -} - -func (p *srcNameNode) setSrcMapLinks(id uint32) { p.srcMapLinks = append(p.srcMapLinks, id) } -func (p *srcNameNode) setScorecardLinks(id uint32) { p.scorecardLinks = append(p.scorecardLinks, id) } -func (p *srcNameNode) setOccurrenceLinks(id uint32) { p.occurrences = append(p.occurrences, id) } -func (p *srcNameNode) setCertifyBadLinks(id uint32) { p.badLinks = append(p.badLinks, id) } -func (p *srcNameNode) setCertifyGoodLinks(id uint32) { p.goodLinks = append(p.goodLinks, id) } -func (p *srcNameNode) setCertifyLegals(id uint32) { p.certifyLegals = append(p.certifyLegals, id) } -func (p *srcNameNode) setHasMetadataLinks(id uint32) { - p.hasMetadataLinks = append(p.hasMetadataLinks, id) -} -func (p *srcNameNode) setPointOfContactLinks(id uint32) { - p.pointOfContactLinks = append(p.pointOfContactLinks, id) -} - -// Ingest Source - -func (c *demoClient) IngestSources(ctx context.Context, sources []*model.SourceInputSpec) ([]*model.Source, error) { - var modelSources []*model.Source - for _, src := range sources { - modelSrc, err := c.IngestSource(ctx, *src) - if err != nil { - return nil, gqlerror.Errorf("IngestSources failed with err: %v", err) - } - modelSources = append(modelSources, modelSrc) - } - return modelSources, nil -} - -func (c *demoClient) IngestSource(ctx context.Context, input model.SourceInputSpec) (*model.Source, error) { - c.m.RLock() - namespacesStruct, hasNamespace := c.sources[input.Type] - c.m.RUnlock() - if !hasNamespace { - c.m.Lock() - namespacesStruct, hasNamespace = c.sources[input.Type] - if !hasNamespace { - namespacesStruct = &srcNamespaceStruct{ - id: c.getNextID(), - typeKey: input.Type, - namespaces: srcNamespaceMap{}, - } - c.index[namespacesStruct.id] = namespacesStruct - c.sources[input.Type] = namespacesStruct - } - c.m.Unlock() - } - namespaces := namespacesStruct.namespaces - - c.m.RLock() - namesStruct, hasName := namespaces[input.Namespace] - c.m.RUnlock() - if !hasName { - c.m.Lock() - namesStruct, hasName = namespaces[input.Namespace] - if !hasName { - namesStruct = &srcNameStruct{ - id: c.getNextID(), - parent: namespacesStruct.id, - namespace: input.Namespace, - names: srcNameList{}, - } - c.index[namesStruct.id] = namesStruct - namespaces[input.Namespace] = namesStruct - } - c.m.Unlock() - } - - c.m.RLock() - duplicate, collectedSrcName := duplicateSrcName(namesStruct.names, input) - c.m.RUnlock() - if !duplicate { - c.m.Lock() - duplicate, collectedSrcName = duplicateSrcName(namesStruct.names, input) - if !duplicate { - collectedSrcName = &srcNameNode{ - id: c.getNextID(), - parent: namesStruct.id, - name: input.Name, - } - c.index[collectedSrcName.id] = collectedSrcName - if input.Tag != nil { - collectedSrcName.tag = nilToEmpty(input.Tag) - } - if input.Commit != nil { - collectedSrcName.commit = nilToEmpty(input.Commit) - } - namesStruct.names = append(namesStruct.names, collectedSrcName) - } - c.m.Unlock() - } - - // build return GraphQL type - c.m.RLock() - defer c.m.RUnlock() - return c.buildSourceResponse(collectedSrcName.id, nil) -} - -func duplicateSrcName(names srcNameList, input model.SourceInputSpec) (bool, *srcNameNode) { - for _, src := range names { - if src.name != input.Name { - continue - } - if noMatchInput(input.Tag, src.tag) { - continue - } - if noMatchInput(input.Commit, src.commit) { - continue - } - return true, src - } - return false, nil -} - -// Query Source - -func (c *demoClient) Sources(ctx context.Context, filter *model.SourceSpec) ([]*model.Source, error) { - c.m.RLock() - defer c.m.RUnlock() - if filter != nil && filter.ID != nil { - id, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - s, err := c.buildSourceResponse(uint32(id), filter) - if err != nil { - if errors.Is(err, errNotFound) { - // not found - return nil, nil - } - return nil, err - } - return []*model.Source{s}, nil - } - - out := []*model.Source{} - if filter != nil && filter.Type != nil { - srcNamespaceStruct, ok := c.sources[*filter.Type] - if ok { - sNamespaces := buildSourceNamespace(srcNamespaceStruct, filter) - if len(sNamespaces) > 0 { - out = append(out, &model.Source{ - ID: nodeID(srcNamespaceStruct.id), - Type: srcNamespaceStruct.typeKey, - Namespaces: sNamespaces, - }) - } - } - } else { - for dbType, srcNamespaceStruct := range c.sources { - sNamespaces := buildSourceNamespace(srcNamespaceStruct, filter) - if len(sNamespaces) > 0 { - out = append(out, &model.Source{ - ID: nodeID(srcNamespaceStruct.id), - Type: dbType, - Namespaces: sNamespaces, - }) - } - } - } - return out, nil -} - -func buildSourceNamespace(srcNamespaceStruct *srcNamespaceStruct, filter *model.SourceSpec) []*model.SourceNamespace { - sNamespaces := []*model.SourceNamespace{} - if filter != nil && filter.Namespace != nil { - srcNameStruct, ok := srcNamespaceStruct.namespaces[*filter.Namespace] - if ok { - sns := buildSourceName(srcNameStruct, filter) - if len(sns) > 0 { - sNamespaces = append(sNamespaces, &model.SourceNamespace{ - ID: nodeID(srcNameStruct.id), - Namespace: srcNameStruct.namespace, - Names: sns, - }) - } - } - } else { - for namespace, srcNameStruct := range srcNamespaceStruct.namespaces { - sns := buildSourceName(srcNameStruct, filter) - if len(sns) > 0 { - sNamespaces = append(sNamespaces, &model.SourceNamespace{ - ID: nodeID(srcNameStruct.id), - Namespace: namespace, - Names: sns, - }) - } - } - } - return sNamespaces -} - -func buildSourceName(srcNameStruct *srcNameStruct, filter *model.SourceSpec) []*model.SourceName { - sns := []*model.SourceName{} - for _, s := range srcNameStruct.names { - if filter != nil && noMatch(filter.Name, s.name) { - continue - } - if filter != nil && noMatch(filter.Tag, s.tag) { - continue - } - if filter != nil && noMatch(filter.Commit, s.commit) { - continue - } - sns = append(sns, &model.SourceName{ - ID: nodeID(s.id), - Name: s.name, - Tag: &s.tag, - Commit: &s.commit, - }) - } - return sns -} - -// Builds a model.Source to send as GraphQL response, starting from id. -// The optional filter allows restricting output (on selection operations). -func (c *demoClient) buildSourceResponse(id uint32, filter *model.SourceSpec) (*model.Source, error) { - if filter != nil && filter.ID != nil { - filteredID, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - if uint32(filteredID) != id { - return nil, nil - } - } - - node, ok := c.index[id] - if !ok { - return nil, fmt.Errorf("%w : ID does not match existing node", errNotFound) - } - - snl := []*model.SourceName{} - if nameNode, ok := node.(*srcNameNode); ok { - if filter != nil && noMatch(filter.Name, nameNode.name) { - return nil, nil - } - if filter != nil && noMatch(filter.Tag, nameNode.tag) { - return nil, nil - } - if filter != nil && noMatch(filter.Commit, nameNode.commit) { - return nil, nil - } - snl = append(snl, &model.SourceName{ - // IDs are generated as string even though we ask for integers - // See https://github.com/99designs/gqlgen/issues/2561 - ID: nodeID(nameNode.id), - Name: nameNode.name, - Tag: &nameNode.tag, - Commit: &nameNode.commit, - }) - node, ok = c.index[nameNode.parent] - if !ok { - return nil, fmt.Errorf("Internal ID does not match existing node") - } - } - - snsl := []*model.SourceNamespace{} - if nameStruct, ok := node.(*srcNameStruct); ok { - if filter != nil && noMatch(filter.Namespace, nameStruct.namespace) { - return nil, nil - } - snsl = append(snsl, &model.SourceNamespace{ - ID: nodeID(nameStruct.id), - Namespace: nameStruct.namespace, - Names: snl, - }) - node, ok = c.index[nameStruct.parent] - if !ok { - return nil, fmt.Errorf("Internal ID does not match existing node") - } - } - - namespaceStruct, ok := node.(*srcNamespaceStruct) - if !ok { - return nil, fmt.Errorf("%w: ID does not match expected node type for source namespace", errNotFound) - } - s := model.Source{ - ID: nodeID(namespaceStruct.id), - Type: namespaceStruct.typeKey, - Namespaces: snsl, - } - if filter != nil && noMatch(filter.Type, s.Type) { - return nil, nil - } - return &s, nil -} - -func getSourceIDFromInput(c *demoClient, input model.SourceInputSpec) (uint32, error) { - srcNamespace, srcHasNamespace := c.sources[input.Type] - if !srcHasNamespace { - return 0, gqlerror.Errorf("Source type \"%s\" not found", input.Type) - } - srcName, srcHasName := srcNamespace.namespaces[input.Namespace] - if !srcHasName { - return 0, gqlerror.Errorf("Source namespace \"%s\" not found", input.Namespace) - } - found := false - var sourceID uint32 - for _, src := range srcName.names { - if src.name != input.Name { - continue - } - if noMatchInput(input.Tag, src.tag) { - continue - } - if noMatchInput(input.Commit, src.commit) { - continue - } - if found { - return 0, gqlerror.Errorf("More than one source matches input") - } - sourceID = src.id - found = true - } - if !found { - return 0, gqlerror.Errorf("No source matches input") - } - return sourceID, nil -} - -func (c *demoClient) exactSource(filter *model.SourceSpec) (*srcNameNode, error) { - if filter == nil { - return nil, nil - } - if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - id := uint32(id64) - if node, ok := c.index[id]; ok { - if s, ok := node.(*srcNameNode); ok { - return s, nil - } - } - } - if filter.Type != nil && filter.Namespace != nil && filter.Name != nil && (filter.Tag != nil || filter.Commit != nil) { - tp, ok := c.sources[*filter.Type] - if !ok { - return nil, nil - } - ns, ok := tp.namespaces[*filter.Namespace] - if !ok { - return nil, nil - } - for _, n := range ns.names { - if *filter.Name != n.name || - noMatchInput(filter.Tag, n.tag) || - noMatchInput(filter.Commit, n.commit) { - continue - } - return n, nil - } - } - return nil, nil -} diff --git a/pkg/assembler/backends/inmem/vulnerability.go b/pkg/assembler/backends/inmem/vulnerability.go deleted file mode 100644 index 6d09192139..0000000000 --- a/pkg/assembler/backends/inmem/vulnerability.go +++ /dev/null @@ -1,356 +0,0 @@ -// -// Copyright 2023 The GUAC Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package inmem - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - - "github.com/vektah/gqlparser/v2/gqlerror" - - "github.com/guacsec/guac/internal/testing/ptrfrom" - "github.com/guacsec/guac/pkg/assembler/graphql/model" -) - -const noVulnType string = "novuln" - -// Internal data: Vulnerability -type vulnTypeMap map[string]*vulnTypeStruct -type vulnTypeStruct struct { - id uint32 - typeKey string - vulnIDs vulnIDList -} -type vulnIDList []*vulnIDNode -type vulnIDNode struct { - id uint32 - parent uint32 - vulnID string - certifyVulnLinks []uint32 - vulnEqualLinks []uint32 - vexLinks []uint32 - vulnMetadataLinks []uint32 -} - -func (n *vulnTypeStruct) ID() uint32 { return n.id } -func (n *vulnIDNode) ID() uint32 { return n.id } - -func (n *vulnTypeStruct) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgeVulnerabilityTypeVulnerabilityID] { - for _, v := range n.vulnIDs { - out = append(out, v.id) - } - } - return out -} -func (n *vulnIDNode) Neighbors(allowedEdges edgeMap) []uint32 { - var out []uint32 - if allowedEdges[model.EdgeVulnerabilityIDVulnerabilityType] { - out = append(out, n.parent) - } - if allowedEdges[model.EdgeVulnerabilityCertifyVuln] { - out = append(out, n.certifyVulnLinks...) - } - if allowedEdges[model.EdgeVulnerabilityVulnEqual] { - out = append(out, n.vulnEqualLinks...) - } - if allowedEdges[model.EdgeVulnerabilityCertifyVexStatement] { - out = append(out, n.vexLinks...) - } - if allowedEdges[model.EdgeVulnMetadataVulnerability] { - out = append(out, n.vulnMetadataLinks...) - } - - return out -} - -func (n *vulnTypeStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildVulnResponse(n.id, nil) -} -func (n *vulnIDNode) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildVulnResponse(n.id, nil) -} - -// certifyVulnerability back edges -func (n *vulnIDNode) setVulnerabilityLinks(id uint32) { - n.certifyVulnLinks = append(n.certifyVulnLinks, id) -} - -// equalVulnerability back edges -func (n *vulnIDNode) setVulnEqualLinks(id uint32) { n.vulnEqualLinks = append(n.vulnEqualLinks, id) } - -// certifyVexStatement back edges -func (n *vulnIDNode) setVexLinks(id uint32) { n.vexLinks = append(n.vexLinks, id) } - -// vulnerability Metadata back edges -func (n *vulnIDNode) setVulnMetadataLinks(id uint32) { - n.vulnMetadataLinks = append(n.vulnMetadataLinks, id) -} - -// Ingest Vulnerabilities - -func (c *demoClient) IngestVulnerabilities(ctx context.Context, vulns []*model.VulnerabilityInputSpec) ([]*model.Vulnerability, error) { - var modelVulnerabilities []*model.Vulnerability - for _, vuln := range vulns { - modelVuln, err := c.IngestVulnerability(ctx, *vuln) - if err != nil { - return nil, gqlerror.Errorf("IngestVulnerability failed with err: %v", err) - } - modelVulnerabilities = append(modelVulnerabilities, modelVuln) - } - return modelVulnerabilities, nil -} - -func (c *demoClient) IngestVulnerability(ctx context.Context, vuln model.VulnerabilityInputSpec) (*model.Vulnerability, error) { - return c.ingestVuln(ctx, vuln, true) -} - -func (c *demoClient) ingestVuln(ctx context.Context, input model.VulnerabilityInputSpec, readOnly bool) (*model.Vulnerability, error) { - typeLowerCase := strings.ToLower(input.Type) - vulIDLoweCase := strings.ToLower(input.VulnerabilityID) - c.m.RLock() - typeStruct, hasType := c.vulnerabilities[typeLowerCase] - c.m.RUnlock() - if !hasType { - c.m.Lock() - typeStruct, hasType = c.vulnerabilities[typeLowerCase] - if !hasType { - typeStruct = &vulnTypeStruct{ - id: c.getNextID(), - typeKey: typeLowerCase, - vulnIDs: vulnIDList{}, - } - c.index[typeStruct.id] = typeStruct - c.vulnerabilities[typeLowerCase] = typeStruct - } - c.m.Unlock() - } - - c.m.RLock() - duplicate, collectedVulnID := duplicateVulnID(typeStruct.vulnIDs, input) - c.m.RUnlock() - if !duplicate { - c.m.Lock() - duplicate, collectedVulnID = duplicateVulnID(typeStruct.vulnIDs, input) - if !duplicate { - collectedVulnID = &vulnIDNode{ - id: c.getNextID(), - parent: typeStruct.id, - vulnID: vulIDLoweCase, - } - c.index[collectedVulnID.id] = collectedVulnID - typeStruct.vulnIDs = append(typeStruct.vulnIDs, collectedVulnID) - } - c.m.Unlock() - } - - // build return GraphQL type - c.m.RLock() - defer c.m.RUnlock() - return c.buildVulnResponse(collectedVulnID.id, nil) -} - -func duplicateVulnID(vulnIDs vulnIDList, input model.VulnerabilityInputSpec) (bool, *vulnIDNode) { - for _, vulnID := range vulnIDs { - if vulnID.vulnID != strings.ToLower(input.VulnerabilityID) { - continue - } - return true, vulnID - } - return false, nil -} - -// Query Vulnerabilities -func (c *demoClient) Vulnerabilities(ctx context.Context, filter *model.VulnerabilitySpec) ([]*model.Vulnerability, error) { - c.m.RLock() - defer c.m.RUnlock() - if filter != nil && filter.ID != nil { - id, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - v, err := c.buildVulnResponse(uint32(id), filter) - if err != nil { - if errors.Is(err, errNotFound) { - // not found - return nil, nil - } - return nil, err - } - return []*model.Vulnerability{v}, nil - } - - if filter.NoVuln != nil && !*filter.NoVuln { - if filter.Type != nil && *filter.Type == noVulnType { - return []*model.Vulnerability{}, gqlerror.Errorf("novuln boolean set to false, cannot specify vulnerability type to be novuln") - } - } - - out := []*model.Vulnerability{} - // if novuln is specified, retrieve all "novuln" type nodes - if filter != nil && filter.NoVuln != nil && *filter.NoVuln { - filter.Type = ptrfrom.String(noVulnType) - filter.VulnerabilityID = ptrfrom.String("") - } - - if filter != nil && filter.Type != nil { - typeStruct, ok := c.vulnerabilities[strings.ToLower(*filter.Type)] - if ok { - vulnIDs := buildVulnID(typeStruct, filter) - if len(vulnIDs) > 0 { - out = append(out, &model.Vulnerability{ - ID: nodeID(typeStruct.id), - Type: typeStruct.typeKey, - VulnerabilityIDs: vulnIDs, - }) - } - } - } else { - for vulnType, typeStruct := range c.vulnerabilities { - vulnIDs := buildVulnID(typeStruct, filter) - if len(vulnIDs) > 0 { - out = append(out, &model.Vulnerability{ - ID: nodeID(typeStruct.id), - Type: vulnType, - VulnerabilityIDs: vulnIDs, - }) - } - } - } - return out, nil -} - -func buildVulnID(typeStruct *vulnTypeStruct, filter *model.VulnerabilitySpec) []*model.VulnerabilityID { - vunIDs := []*model.VulnerabilityID{} - for _, v := range typeStruct.vulnIDs { - if filter != nil && noMatch(toLower(filter.VulnerabilityID), v.vulnID) { - continue - } - vunIDs = append(vunIDs, &model.VulnerabilityID{ - ID: nodeID(v.id), - VulnerabilityID: v.vulnID, - }) - } - return vunIDs -} - -func (c *demoClient) exactVulnerability(filter *model.VulnerabilitySpec) (*vulnIDNode, error) { - if filter == nil { - return nil, nil - } - if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - id := uint32(id64) - if node, ok := c.index[id]; ok { - if v, ok := node.(*vulnIDNode); ok { - return v, nil - } - } - } - if filter.Type != nil && filter.VulnerabilityID != nil { - tp, ok := c.vulnerabilities[strings.ToLower(*filter.Type)] - if !ok { - return nil, nil - } - for _, vulnID := range tp.vulnIDs { - if strings.ToLower(*filter.VulnerabilityID) != vulnID.vulnID { - continue - } - return vulnID, nil - } - } - return nil, nil -} - -// Builds a model.Vulnerability to send as GraphQL response, starting from id. -// The optional filter allows restricting output (on selection operations). -func (c *demoClient) buildVulnResponse(id uint32, filter *model.VulnerabilitySpec) (*model.Vulnerability, error) { - if filter != nil && filter.ID != nil { - filteredID, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err - } - if uint32(filteredID) != id { - return nil, nil - } - } - - node, ok := c.index[id] - if !ok { - return nil, fmt.Errorf("%w : ID does not match existing node", errNotFound) - } - - var vl []*model.VulnerabilityID - if vulnNode, ok := node.(*vulnIDNode); ok { - if filter != nil && noMatch(toLower(filter.VulnerabilityID), vulnNode.vulnID) { - return nil, nil - } - vl = append(vl, &model.VulnerabilityID{ - // IDs are generated as string even though we ask for integers - // See https://github.com/99designs/gqlgen/issues/2561 - ID: nodeID(vulnNode.id), - VulnerabilityID: vulnNode.vulnID, - }) - node, ok = c.index[vulnNode.parent] - if !ok { - return nil, fmt.Errorf("internal ID does not match existing node") - } - } - - typeStruct, ok := node.(*vulnTypeStruct) - if !ok { - return nil, fmt.Errorf("%w: ID does not match expected node type for vulnerability", errNotFound) - } - v := model.Vulnerability{ - ID: nodeID(typeStruct.id), - Type: typeStruct.typeKey, - VulnerabilityIDs: vl, - } - if filter != nil && noMatch(toLower(filter.Type), v.Type) { - return nil, nil - } - return &v, nil -} - -func getVulnerabilityIDFromInput(c *demoClient, input model.VulnerabilityInputSpec) (uint32, error) { - typeStruct, vulnTypeFound := c.vulnerabilities[strings.ToLower(input.Type)] - if !vulnTypeFound { - return 0, gqlerror.Errorf("vulnerability type \"%s\" not found", input.Type) - } - found := false - var vulnNodeID uint32 - for _, vulnID := range typeStruct.vulnIDs { - if vulnID.vulnID != strings.ToLower(input.VulnerabilityID) { - continue - } - if found { - return 0, gqlerror.Errorf("more than one vulnerability matches input") - } - vulnNodeID = vulnID.id - found = true - } - if !found { - return 0, gqlerror.Errorf("No vulnerability matches input") - } - return vulnNodeID, nil -} diff --git a/pkg/assembler/backends/keyvalue/artifact.go b/pkg/assembler/backends/keyvalue/artifact.go new file mode 100644 index 0000000000..cb46b19311 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/artifact.go @@ -0,0 +1,302 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal data: Artifacts +type artStruct struct { + ThisID string + Algorithm string + Digest string + HashEquals []string + Occurrences []string + HasSBOMs []string + HasSLSAs []string + VexLinks []string + BadLinks []string + GoodLinks []string + HasMetadataLinks []string + PointOfContactLinks []string +} + +func (n *artStruct) ID() string { return n.ThisID } + +func (n *artStruct) Neighbors(allowedEdges edgeMap) []string { + out := []string{} + if allowedEdges[model.EdgeArtifactHashEqual] { + out = append(out, n.HashEquals...) + } + if allowedEdges[model.EdgeArtifactIsOccurrence] { + out = append(out, n.Occurrences...) + } + if allowedEdges[model.EdgeArtifactHasSbom] { + out = append(out, n.HasSBOMs...) + } + if allowedEdges[model.EdgeArtifactHasSlsa] { + out = append(out, n.HasSLSAs...) + } + if allowedEdges[model.EdgeArtifactCertifyVexStatement] { + out = append(out, n.VexLinks...) + } + if allowedEdges[model.EdgeArtifactCertifyBad] { + out = append(out, n.BadLinks...) + } + if allowedEdges[model.EdgeArtifactCertifyGood] { + out = append(out, n.GoodLinks...) + } + if allowedEdges[model.EdgeArtifactHasMetadata] { + out = append(out, n.HasMetadataLinks...) + } + if allowedEdges[model.EdgeArtifactPointOfContact] { + out = append(out, n.PointOfContactLinks...) + } + + return out +} + +func (n *artStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convArtifact(n), nil +} + +func (n *artStruct) setOccurrences(ctx context.Context, ID string, c *demoClient) error { + n.Occurrences = append(n.Occurrences, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setHashEquals(ctx context.Context, ID string, c *demoClient) error { + n.HashEquals = append(n.HashEquals, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setHasSBOMs(ctx context.Context, ID string, c *demoClient) error { + n.HasSBOMs = append(n.HasSBOMs, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setHasSLSAs(ctx context.Context, ID string, c *demoClient) error { + n.HasSLSAs = append(n.HasSLSAs, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setVexLinks(ctx context.Context, ID string, c *demoClient) error { + n.VexLinks = append(n.VexLinks, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setCertifyBadLinks(ctx context.Context, ID string, c *demoClient) error { + n.BadLinks = append(n.BadLinks, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setCertifyGoodLinks(ctx context.Context, ID string, c *demoClient) error { + n.GoodLinks = append(n.GoodLinks, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setHasMetadataLinks(ctx context.Context, ID string, c *demoClient) error { + n.HasMetadataLinks = append(n.HasMetadataLinks, ID) + return setkv(ctx, artCol, n, c) +} +func (n *artStruct) setPointOfContactLinks(ctx context.Context, ID string, c *demoClient) error { + n.PointOfContactLinks = append(n.PointOfContactLinks, ID) + return setkv(ctx, artCol, n, c) +} + +func (n *artStruct) Key() string { + return strings.Join([]string{n.Algorithm, n.Digest}, ":") +} + +func (c *demoClient) artifactByInput(ctx context.Context, a *model.ArtifactInputSpec) (*artStruct, error) { + inA := &artStruct{ + Algorithm: strings.ToLower(a.Algorithm), + Digest: strings.ToLower(a.Digest), + } + return byKeykv[*artStruct](ctx, artCol, inA.Key(), c) +} + +func (c *demoClient) artifactModelByID(ctx context.Context, id string) (*model.Artifact, error) { + a, err := byIDkv[*artStruct](ctx, id, c) + if err != nil { + return nil, err + } + return c.convArtifact(a), nil +} + +// Ingest Artifacts + +func (c *demoClient) IngestArtifacts(ctx context.Context, artifacts []*model.ArtifactInputSpec) ([]*model.Artifact, error) { + var modelArtifacts []*model.Artifact + for _, art := range artifacts { + modelArt, err := c.IngestArtifact(ctx, art) + if err != nil { + return nil, gqlerror.Errorf("ingestArtifact failed with err: %v", err) + } + modelArtifacts = append(modelArtifacts, modelArt) + } + return modelArtifacts, nil +} + +func (c *demoClient) IngestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec) (*model.Artifact, error) { + return c.ingestArtifact(ctx, artifact, true) +} + +func (c *demoClient) ingestArtifact(ctx context.Context, artifact *model.ArtifactInputSpec, readOnly bool) (*model.Artifact, error) { + algorithm := strings.ToLower(artifact.Algorithm) + digest := strings.ToLower(artifact.Digest) + + inA := &artStruct{ + Algorithm: algorithm, + Digest: digest, + } + + lock(&c.m, readOnly) + defer unlock(&c.m, readOnly) + + outA, err := byKeykv[*artStruct](ctx, artCol, inA.Key(), c) + + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + // Got KeyError: not found, so do insert + if readOnly { + c.m.RUnlock() + a, err := c.ingestArtifact(ctx, artifact, false) + c.m.RLock() // relock so that defer unlock does not panic + return a, err + } + inA.ThisID = c.getNextID() + if err := c.addToIndex(ctx, artCol, inA); err != nil { + return nil, err + } + if err := setkv(ctx, artCol, inA, c); err != nil { + return nil, err + } + outA = inA + } + + return c.convArtifact(outA), nil +} + +func (c *demoClient) artifactExact(ctx context.Context, artifactSpec *model.ArtifactSpec) (*artStruct, error) { + algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm)) + digest := strings.ToLower(nilToEmpty(artifactSpec.Digest)) + + // If ID is provided, try to look up + if artifactSpec.ID != nil { + a, err := byIDkv[*artStruct](ctx, *artifactSpec.ID, c) + if err != nil { + // Not found + return nil, nil + } + // If found by ID, ignore rest of fields in spec and return as a match + return a, nil + } + + // If algo and digest are provied, try to lookup + if algorithm != "" && digest != "" { + inA := &artStruct{ + Algorithm: algorithm, + Digest: digest, + } + if outA, err := byKeykv[*artStruct](ctx, artCol, inA.Key(), c); err != nil { + return outA, nil + } + } + return nil, nil +} + +// Query Artifacts + +func (c *demoClient) Artifacts(ctx context.Context, artifactSpec *model.ArtifactSpec) ([]*model.Artifact, error) { + c.m.RLock() + defer c.m.RUnlock() + a, err := c.artifactExact(ctx, artifactSpec) + if err != nil { + return nil, gqlerror.Errorf("Artifacts :: invalid spec %s", err) + } + if a != nil { + return []*model.Artifact{c.convArtifact(a)}, nil + } + + algorithm := strings.ToLower(nilToEmpty(artifactSpec.Algorithm)) + digest := strings.ToLower(nilToEmpty(artifactSpec.Digest)) + var rv []*model.Artifact + artKeys, err := c.kv.Keys(ctx, artCol) + if err != nil { + return nil, err + } + for _, ak := range artKeys { + a, err := byKeykv[*artStruct](ctx, artCol, ak, c) + if err != nil { + return nil, err + } + + matchAlgorithm := false + if algorithm == "" || algorithm == a.Algorithm { + matchAlgorithm = true + } + + matchDigest := false + if digest == "" || digest == a.Digest { + matchDigest = true + } + + if matchDigest && matchAlgorithm { + rv = append(rv, c.convArtifact(a)) + } + } + return rv, nil +} + +func (c *demoClient) convArtifact(a *artStruct) *model.Artifact { + return &model.Artifact{ + ID: a.ThisID, + Digest: a.Digest, + Algorithm: a.Algorithm, + } +} + +// Builds a model.Artifact to send as GraphQL response, starting from ID. +// The optional filter allows restricting output (on selection operations). +func (c *demoClient) buildArtifactResponse(ctx context.Context, ID string, filter *model.ArtifactSpec) (*model.Artifact, error) { + if filter != nil && filter.ID != nil && *filter.ID != ID { + return nil, nil + } + + artNode, err := byIDkv[*artStruct](ctx, ID, c) + if err != nil { + return nil, fmt.Errorf("ID does not match expected node type for artifact, %w", err) + } + + if filter != nil && noMatch(toLower(filter.Algorithm), artNode.Algorithm) { + return nil, nil + } + if filter != nil && noMatch(toLower(filter.Digest), artNode.Digest) { + return nil, nil + } + art := &model.Artifact{ + ID: artNode.ThisID, + Algorithm: artNode.Algorithm, + Digest: artNode.Digest, + } + + return art, nil +} diff --git a/pkg/assembler/backends/inmem/artifact_test.go b/pkg/assembler/backends/keyvalue/artifact_test.go similarity index 85% rename from pkg/assembler/backends/inmem/artifact_test.go rename to pkg/assembler/backends/keyvalue/artifact_test.go index d688175cb4..f730f3950e 100644 --- a/pkg/assembler/backends/inmem/artifact_test.go +++ b/pkg/assembler/backends/keyvalue/artifact_test.go @@ -13,12 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" "reflect" - "strconv" "strings" "testing" @@ -30,17 +29,17 @@ import ( func Test_artifactStruct_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &artStruct{ - id: tt.id, + ThisID: tt.id, } if got := b.ID(); got != tt.want { t.Errorf("builderStruct.ID() = %v, want %v", got, tt.want) @@ -51,76 +50,76 @@ func Test_artifactStruct_ID(t *testing.T) { func Test_artifactStruct_Neighbors(t *testing.T) { type fields struct { - id uint32 + id string algorithm string digest string - hashEquals []uint32 - occurrences []uint32 - hasSLSAs []uint32 - vexLinks []uint32 - badLinks []uint32 - goodLinks []uint32 + hashEquals []string + occurrences []string + hasSLSAs []string + vexLinks []string + badLinks []string + goodLinks []string } tests := []struct { name string fields fields allowedEdges edgeMap - want []uint32 + want []string }{{ name: "hashEquals", fields: fields{ - hashEquals: []uint32{343, 546}, + hashEquals: []string{"343", "546"}, }, allowedEdges: edgeMap{model.EdgeArtifactHashEqual: true}, - want: []uint32{343, 546}, + want: []string{"343", "546"}, }, { name: "occurrences", fields: fields{ - occurrences: []uint32{2324, 1234}, + occurrences: []string{"2324", "1234"}, }, allowedEdges: edgeMap{model.EdgeArtifactIsOccurrence: true}, - want: []uint32{2324, 1234}, + want: []string{"2324", "1234"}, }, { name: "hasSLSAs", fields: fields{ - hasSLSAs: []uint32{445, 1232244}, + hasSLSAs: []string{"445", "1232244"}, }, allowedEdges: edgeMap{model.EdgeArtifactHasSlsa: true}, - want: []uint32{445, 1232244}, + want: []string{"445", "1232244"}, }, { name: "vexLinks", fields: fields{ - vexLinks: []uint32{987, 9876}, + vexLinks: []string{"987", "9876"}, }, allowedEdges: edgeMap{model.EdgeArtifactCertifyVexStatement: true}, - want: []uint32{987, 9876}, + want: []string{"987", "9876"}, }, { name: "badLinks", fields: fields{ - badLinks: []uint32{5322, 544}, + badLinks: []string{"5322", "544"}, }, allowedEdges: edgeMap{model.EdgeArtifactCertifyBad: true}, - want: []uint32{5322, 544}, + want: []string{"5322", "544"}, }, { name: "goodLinks", fields: fields{ - goodLinks: []uint32{25468, 1458}, + goodLinks: []string{"25468", "1458"}, }, allowedEdges: edgeMap{model.EdgeArtifactCertifyGood: true}, - want: []uint32{25468, 1458}, + want: []string{"25468", "1458"}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { a := &artStruct{ - id: tt.fields.id, - algorithm: tt.fields.algorithm, - digest: tt.fields.digest, - hashEquals: tt.fields.hashEquals, - occurrences: tt.fields.occurrences, - hasSLSAs: tt.fields.hasSLSAs, - vexLinks: tt.fields.vexLinks, - badLinks: tt.fields.badLinks, - goodLinks: tt.fields.goodLinks, + ThisID: tt.fields.id, + Algorithm: tt.fields.algorithm, + Digest: tt.fields.digest, + HashEquals: tt.fields.hashEquals, + Occurrences: tt.fields.occurrences, + HasSLSAs: tt.fields.hasSLSAs, + VexLinks: tt.fields.vexLinks, + BadLinks: tt.fields.badLinks, + GoodLinks: tt.fields.goodLinks, } if got := a.Neighbors(tt.allowedEdges); !reflect.DeepEqual(got, tt.want) { t.Errorf("builderStruct.Neighbors() = %v, want %v", got, tt.want) @@ -131,7 +130,7 @@ func Test_artifactStruct_Neighbors(t *testing.T) { func Test_artifactStruct_BuildModelNode(t *testing.T) { type fields struct { - id uint32 + id string algorithm string digest string } @@ -143,12 +142,12 @@ func Test_artifactStruct_BuildModelNode(t *testing.T) { }{{ name: "sha256", fields: fields{ - id: uint32(43), + id: "43", algorithm: strings.ToLower("sha256"), digest: strings.ToLower("6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf"), }, want: &model.Artifact{ - ID: nodeID(uint32(43)), + ID: "43", Algorithm: strings.ToLower("sha256"), Digest: strings.ToLower("6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf"), }, @@ -156,12 +155,12 @@ func Test_artifactStruct_BuildModelNode(t *testing.T) { }, { name: "sha1", fields: fields{ - id: uint32(53), + id: "53", algorithm: strings.ToLower("sha1"), digest: strings.ToLower("7A8F47318E4676DACB0142AFA0B83029CD7BEFD9"), }, want: &model.Artifact{ - ID: nodeID(uint32(53)), + ID: "53", Algorithm: strings.ToLower("sha1"), Digest: strings.ToLower("7a8f47318e4676dacb0142afa0b83029cd7befd9"), }, @@ -169,12 +168,12 @@ func Test_artifactStruct_BuildModelNode(t *testing.T) { }, { name: "sha512", fields: fields{ - id: uint32(63), + id: "63", algorithm: strings.ToLower("sha512"), digest: strings.ToLower("374AB8F711235830769AA5F0B31CE9B72C5670074B34CB302CDAFE3B606233EE92EE01E298E5701F15CC7087714CD9ABD7DDB838A6E1206B3642DE16D9FC9DD7"), }, want: &model.Artifact{ - ID: nodeID(uint32(63)), + ID: "63", Algorithm: strings.ToLower("sha512"), Digest: strings.ToLower("374ab8f711235830769aa5f0b31ce9b72c5670074b34cb302cdafe3b606233ee92ee01e298e5701f15cc7087714cd9abd7ddb838a6e1206b3642de16d9fc9dd7"), }, @@ -182,16 +181,14 @@ func Test_artifactStruct_BuildModelNode(t *testing.T) { }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } + c, _ := getBackend(context.Background(), nil) a := &artStruct{ - id: tt.fields.id, - algorithm: tt.fields.algorithm, - digest: tt.fields.digest, + ThisID: tt.fields.id, + Algorithm: tt.fields.algorithm, + Digest: tt.fields.digest, } - got, err := a.BuildModelNode(c) + b := c.(*demoClient) + got, err := a.BuildModelNode(context.Background(), b) if (err != nil) != tt.wantErr { t.Errorf("artStruct.BuildModelNode() error = %v, wantErr %v", err, tt.wantErr) return @@ -240,11 +237,7 @@ func Test_demoClient_IngestArtifacts(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } - + c, _ := getBackend(ctx, nil) got, err := c.IngestArtifacts(ctx, tt.artifactInputs) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr) @@ -304,10 +297,7 @@ func Test_demoClient_IngestArtifact(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) got, err := c.IngestArtifact(ctx, tt.artifactInput) if (err != nil) != tt.wantErr { @@ -382,10 +372,7 @@ func Test_demoClient_Artifacts(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) ingestedArt, err := c.IngestArtifact(ctx, tt.artifactInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr) @@ -467,24 +454,17 @@ func Test_demoClient_buildArtifactResponse(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) art, err := c.IngestArtifact(ctx, tt.artifactInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr) return } - artID, err := strconv.ParseUint(art.ID, 10, 32) - if err != nil { - t.Errorf("failed to convert string to uint, error = %v", err) - return - } if tt.idInFilter { tt.artifactSpec.ID = &art.ID } - got, err := c.buildArtifactResponse(uint32(artID), tt.artifactSpec) + b := c.(*demoClient) + got, err := b.buildArtifactResponse(context.Background(), art.ID, tt.artifactSpec) if (err != nil) != tt.wantErr { t.Errorf("demoClient.Artifacts() error = %v, wantErr %v", err, tt.wantErr) return @@ -529,26 +509,19 @@ func Test_demoClient_getArtifactIDFromInput(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - artifacts: artMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) art, err := c.IngestArtifact(ctx, tt.artifactInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestArtifact() error = %v, wantErr %v", err, tt.wantErr) return } - artID, err := strconv.ParseUint(art.ID, 10, 32) - if err != nil { - t.Errorf("failed to convert string to uint, error = %v", err) - return - } - got, err := getArtifactIDFromInput(c, *tt.artifactInput) + b := c.(*demoClient) + got, err := b.artifactByInput(context.Background(), tt.artifactInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.Artifacts() error = %v, wantErr %v", err, tt.wantErr) return } - if diff := cmp.Diff(uint32(artID), got, ignoreID); diff != "" { + if diff := cmp.Diff(art.ID, got.ThisID, ignoreID); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } }) diff --git a/pkg/assembler/backends/keyvalue/backend.go b/pkg/assembler/backends/keyvalue/backend.go new file mode 100644 index 0000000000..aed4a93f4a --- /dev/null +++ b/pkg/assembler/backends/keyvalue/backend.go @@ -0,0 +1,377 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "math" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/guacsec/guac/pkg/assembler/backends" + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" + "github.com/guacsec/guac/pkg/assembler/kv/memmap" +) + +func init() { + backends.Register("keyvalue", getBackend) +} + +// node is the common interface of all backend nodes. +type node interface { + // ID provides global IDs for all nodes that can be referenced from + // other places in GUAC. + // + // Since we always ingest data and never remove, + // we can keep this global and increment it as needed. + // + // For fast retrieval, we also keep a map from ID from nodes that have + // it. + // + // IDs are stored as string in graphql even though we ask for integers + // See https://github.com/99designs/gqlgen/issues/2561 + ID() string + + // Neighbors allows retrieving neighbors of a node using the backlinks. + // + // This is useful for path related queries where the type of the node + // is not as relevant as its connections. + // + // The allowedEdges argument allows filtering the set of neighbors to + // only include certain GUAC verbs. + Neighbors(allowedEdges edgeMap) []string + + // BuildModelNode builds a GraphQL return type for a backend node, + BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) + + Key() string +} + +//type indexType map[string]node + +var errNotFound = errors.New("not found") +var errTypeNotMatch = errors.New("Stored type does not match") + +// Scorecard scores are in range of 1-10, so a single step at 100 should be +// plenty big +var epsilon = math.Nextafter(100, 100.1) - 100 + +const ( + // Collection names must not have ":" in them + indexCol = "index" + artCol = "artifacts" + occCol = "isOccurrences" + pkgTypeCol = "pkgTypes" + pkgNSCol = "pkgNamespaces" + pkgNameCol = "pkgNames" + pkgVerCol = "pkgVersions" + isDepCol = "isDependencies" + hasMDCol = "hasMetadatas" + hasSBOMCol = "hasSBOMs" + srcTypeCol = "srcTypes" + srcNSCol = "srcNamespaces" + srcNameCol = "srcNames" + cgCol = "certifyGoods" + cbCol = "certifyBads" + builderCol = "builders" + licenseCol = "licenses" + clCol = "certifyLegals" + cscCol = "certifyScorecards" + slsaCol = "hasSLSAs" + hsaCol = "hasSourceAts" + hashEqCol = "hashEquals" + pkgEqCol = "pkgEquals" + pocCol = "pointOfContacts" + vulnTypeCol = "vulnTypes" + vulnIDCol = "vulnIDs" + vulnEqCol = "vulnEquals" + vulnMDCol = "vulnMetadatas" + cVEXCol = "certifyVEXs" + cVulnCol = "certifyVulns" +) + +func typeColMap(col string) node { + switch col { + case artCol: + return &artStruct{} + case occCol: + return &isOccurrenceStruct{} + case pkgTypeCol: + return &pkgType{} + case pkgNSCol: + return &pkgNamespace{} + case pkgNameCol: + return &pkgName{} + case pkgVerCol: + return &pkgVersion{} + case isDepCol: + return &isDependencyLink{} + case hasMDCol: + return &hasMetadataLink{} + case hasSBOMCol: + return &hasSBOMStruct{} + case srcTypeCol: + return &srcType{} + case srcNSCol: + return &srcNamespace{} + case srcNameCol: + return &srcNameNode{} + case cgCol: + return &goodLink{} + case cbCol: + return &badLink{} + case builderCol: + return &builderStruct{} + case licenseCol: + return &licStruct{} + case clCol: + return &certifyLegalStruct{} + case cscCol: + return &scorecardLink{} + case slsaCol: + return &hasSLSAStruct{} + case hsaCol: + return &srcMapLink{} + case hashEqCol: + return &hashEqualStruct{} + case pkgEqCol: + return &pkgEqualStruct{} + case pocCol: + return &pointOfContactLink{} + case vulnTypeCol: + return &vulnTypeStruct{} + case vulnIDCol: + return &vulnIDNode{} + case vulnEqCol: + return &vulnerabilityEqualLink{} + case vulnMDCol: + return &vulnerabilityMetadataLink{} + case cVEXCol: + return &vexLink{} + case cVulnCol: + return &certifyVulnerabilityLink{} + } + //? + return &artStruct{} +} + +// atomic add to ensure ID is not duplicated +func (c *demoClient) getNextID() string { + atomic.AddUint32(&c.id, 1) + return fmt.Sprintf("%d", c.id) +} + +type demoClient struct { + id uint32 + m sync.RWMutex + kv kv.Store +} + +func getBackend(ctx context.Context, opts backends.BackendArgs) (backends.Backend, error) { + + store, ok := opts.(kv.Store) + if !ok { + store = memmap.GetStore() + } + //kv, err := tikv.GetStore(ctx) + // kv, err := memmap.GetStore() + // if err != nil { + // return nil, err + // } + return &demoClient{ + //kv: &redis.Store{}, + kv: store, + //kv: kv, + }, nil +} + +func noMatch(filter *string, value string) bool { + if filter != nil { + return value != *filter + } + return false +} + +// func noMatchInput(filter *string, value string) bool { +// if filter != nil { +// return value != *filter +// } +// return value != "" +// } + +func nilToEmpty(input *string) string { + if input == nil { + return "" + } + return *input +} + +// func timePtrEqual(a, b *time.Time) bool { +// if a == nil && b == nil { +// return true +// } +// if a != nil && b != nil { +// return a.Equal(*b) +// } +// return false +// } + +func toLower(filter *string) *string { + if filter != nil { + lower := strings.ToLower(*filter) + return &lower + } + return nil +} + +func noMatchFloat(filter *float64, value float64) bool { + if filter != nil { + return math.Abs(*filter-value) > epsilon + } + return false +} + +// func floatEqual(x float64, y float64) bool { +// return math.Abs(x-y) < epsilon +// } + +// delete this +// func byID[E node](id string, c *demoClient) (E, error) { +// var nl E +// o, ok := c.index[id] +// if !ok { +// return nl, fmt.Errorf("%w : id not in index", errNotFound) +// } +// s, ok := o.(E) +// if !ok { +// return nl, fmt.Errorf("%w : node not a %T", errNotFound, nl) +// } +// return s, nil +// } + +func byIDkv[E node](ctx context.Context, id string, c *demoClient) (E, error) { + var nl E + var k string + if err := c.kv.Get(ctx, indexCol, id, &k); err != nil { + return nl, fmt.Errorf("%w : id not found in index %q", err, id) + } + sub := strings.SplitN(k, ":", 2) + if len(sub) != 2 { + return nl, fmt.Errorf("Bad value was stored in index map: %v", k) + } + return byKeykv[E](ctx, sub[0], sub[1], c) +} + +func byKeykv[E node](ctx context.Context, coll string, k string, c *demoClient) (E, error) { + var nl E + if err := validateType(nl, coll); err != nil { + return nl, err + } + err := c.kv.Get(ctx, coll, k, &nl) + return nl, err +} + +func setkv(ctx context.Context, coll string, n node, c *demoClient) error { + // validate type? + return c.kv.Set(ctx, coll, n.Key(), n) +} + +func (c *demoClient) addToIndex(ctx context.Context, coll string, n node) error { + if err := validateType(n, coll); err != nil { + return err + } + val := strings.Join([]string{coll, n.Key()}, ":") + return c.kv.Set(ctx, indexCol, n.ID(), val) +} + +func validateType[E node](n E, c string) error { + if reflect.TypeOf(typeColMap(c)) == reflect.TypeOf(n) { + return nil + } + return fmt.Errorf("%w : found: %q want: %q", errTypeNotMatch, + reflect.TypeOf(typeColMap(c)), reflect.TypeOf(n)) +} + +func lock(m *sync.RWMutex, readOnly bool) { + if readOnly { + m.RLock() + } else { + m.Lock() + } +} + +func unlock(m *sync.RWMutex, readOnly bool) { + if readOnly { + m.RUnlock() + } else { + m.Unlock() + } +} + +func timeKey(t time.Time) string { + return fmt.Sprint(t.Unix()) +} + +func sortAndRemoveDups(ids []string) []string { + numIDs := len(ids) + if numIDs > 1 { + slices.Sort(ids) + nextIndex := 1 + for index := 1; index < numIDs; index++ { + currentVal := ids[index] + if ids[index-1] != currentVal { + ids[nextIndex] = currentVal + nextIndex++ + } + } + ids = ids[:nextIndex] + } + return ids +} + +// IDs should be sorted +func (c *demoClient) isIDPresent(id string, linkIDs []string) bool { + _, found := slices.BinarySearch[[]string](linkIDs, id) + return found +} + +func (c *demoClient) getPackageVersionAndArtifacts(ctx context.Context, pkgOrArt []string) ([]string, []string, error) { + var pkgs []string + var arts []string + for _, id := range pkgOrArt { + found := false + if _, err := byIDkv[*pkgVersion](ctx, id, c); err == nil { + pkgs = append(pkgs, id) + found = true + } + if _, err := byIDkv[*artStruct](ctx, id, c); err == nil { + arts = append(arts, id) + found = true + } + if !found { + return nil, nil, fmt.Errorf("unexpected type in package or artifact list") + } + } + + return pkgs, arts, nil +} diff --git a/pkg/assembler/backends/inmem/builder.go b/pkg/assembler/backends/keyvalue/builder.go similarity index 51% rename from pkg/assembler/backends/inmem/builder.go rename to pkg/assembler/backends/keyvalue/builder.go index 16aefe9d32..70ba76fe10 100644 --- a/pkg/assembler/backends/inmem/builder.go +++ b/pkg/assembler/backends/keyvalue/builder.go @@ -13,45 +13,51 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" "errors" - "strconv" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) -type builderMap map[string]*builderStruct type builderStruct struct { - id uint32 - uri string - hasSLSAs []uint32 + ThisID string + URI string + HasSLSAs []string } -func (b *builderStruct) ID() uint32 { return b.id } +func (n *builderStruct) Key() string { + return n.URI +} + +func (b *builderStruct) ID() string { return b.ThisID } -func (b *builderStruct) Neighbors(allowedEdges edgeMap) []uint32 { +func (b *builderStruct) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgeBuilderHasSlsa] { - return b.hasSLSAs + return b.HasSLSAs } - return []uint32{} + return []string{} } -func (b *builderStruct) BuildModelNode(c *demoClient) (model.Node, error) { +func (b *builderStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { return c.convBuilder(b), nil } -func (n *builderStruct) setHasSLSAs(id uint32) { n.hasSLSAs = append(n.hasSLSAs, id) } +func (n *builderStruct) setHasSLSAs(ctx context.Context, id string, c *demoClient) error { + n.HasSLSAs = append(n.HasSLSAs, id) + return setkv(ctx, builderCol, n, c) +} -func (c *demoClient) builderByKey(uri string) (*builderStruct, error) { - if b, ok := c.builders[uri]; ok { - return b, nil +func (c *demoClient) builderByInput(ctx context.Context, b *model.BuilderInputSpec) (*builderStruct, error) { + in := &builderStruct{ + URI: b.URI, } - return nil, errors.New("builder not found") + return byKeykv[*builderStruct](ctx, builderCol, in.Key(), c) } // Ingest Builders @@ -75,52 +81,58 @@ func (c *demoClient) IngestBuilder(ctx context.Context, builder *model.BuilderIn } func (c *demoClient) ingestBuilder(ctx context.Context, builder *model.BuilderInputSpec, readOnly bool) (*model.Builder, error) { + in := &builderStruct{ + URI: builder.URI, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - b, err := c.builderByKey(builder.URI) - if err != nil { - if readOnly { - c.m.RUnlock() - b, err := c.ingestBuilder(ctx, builder, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - b = &builderStruct{ - id: c.getNextID(), - uri: builder.URI, - } - c.index[b.id] = b - c.builders[builder.URI] = b + out, err := byKeykv[*builderStruct](ctx, builderCol, in.Key(), c) + if err == nil { + return c.convBuilder(out), nil } - return c.convBuilder(b), nil + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + if readOnly { + c.m.RUnlock() + b, err := c.ingestBuilder(ctx, builder, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err + } + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, builderCol, in); err != nil { + return nil, err + } + if err := setkv(ctx, builderCol, in, c); err != nil { + return nil, err + } + + return c.convBuilder(in), nil } // Query Builder func (c *demoClient) Builders(ctx context.Context, builderSpec *model.BuilderSpec) ([]*model.Builder, error) { c.m.RLock() defer c.m.RUnlock() - if builderSpec.ID != nil { - id64, err := strconv.ParseUint(*builderSpec.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("Builders :: couldn't parse id %v", err) - } - id := uint32(id64) - b, err := byID[*builderStruct](id, c) - if err != nil { - return nil, nil - } - return []*model.Builder{c.convBuilder(b)}, nil + b, err := c.exactBuilder(ctx, builderSpec) + if err != nil { + return nil, err } - if builderSpec.URI != nil { - b, err := c.builderByKey(*builderSpec.URI) - if err != nil { - return nil, nil - } + if b != nil { return []*model.Builder{c.convBuilder(b)}, nil } var builders []*model.Builder - for _, b := range c.builders { + bKeys, err := c.kv.Keys(ctx, builderCol) + if err != nil { + return nil, err + } + for _, bk := range bKeys { + b, err := byKeykv[*builderStruct](ctx, builderCol, bk, c) + if err != nil { + return nil, err + } builders = append(builders, c.convBuilder(b)) } return builders, nil @@ -128,30 +140,36 @@ func (c *demoClient) Builders(ctx context.Context, builderSpec *model.BuilderSpe func (c *demoClient) convBuilder(b *builderStruct) *model.Builder { return &model.Builder{ - ID: nodeID(b.id), - URI: b.uri, + ID: b.ThisID, + URI: b.URI, } } -func (c *demoClient) exactBuilder(filter *model.BuilderSpec) (*builderStruct, error) { +func (c *demoClient) exactBuilder(ctx context.Context, filter *model.BuilderSpec) (*builderStruct, error) { if filter == nil { return nil, nil } if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, err + b, err := byIDkv[*builderStruct](ctx, *filter.ID, c) + if err == nil { + return b, nil } - id := uint32(id64) - if node, ok := c.index[id]; ok { - if b, ok := node.(*builderStruct); ok { - return b, nil - } + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err } + // id not found + return nil, nil } if filter.URI != nil { - if b, ok := c.builders[*filter.URI]; ok { - return b, nil + in := &builderStruct{ + URI: *filter.URI, + } + out, err := byKeykv[*builderStruct](ctx, builderCol, in.Key(), c) + if err == nil { + return out, nil + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } } return nil, nil diff --git a/pkg/assembler/backends/inmem/builder_test.go b/pkg/assembler/backends/keyvalue/builder_test.go similarity index 86% rename from pkg/assembler/backends/inmem/builder_test.go rename to pkg/assembler/backends/keyvalue/builder_test.go index 18f256f48e..67dade6c28 100644 --- a/pkg/assembler/backends/inmem/builder_test.go +++ b/pkg/assembler/backends/keyvalue/builder_test.go @@ -13,12 +13,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" "reflect" - "strconv" "strings" "testing" @@ -30,17 +29,17 @@ import ( func Test_builderStruct_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &builderStruct{ - id: tt.id, + ThisID: tt.id, } if got := b.ID(); got != tt.want { t.Errorf("builderStruct.ID() = %v, want %v", got, tt.want) @@ -51,29 +50,29 @@ func Test_builderStruct_ID(t *testing.T) { func Test_builderStruct_Neighbors(t *testing.T) { type fields struct { - id uint32 + id string uri string - hasSLSAs []uint32 + hasSLSAs []string } tests := []struct { name string fields fields allowedEdges edgeMap - want []uint32 + want []string }{{ name: "hasSLSAs", fields: fields{ - hasSLSAs: []uint32{445, 1232244}, + hasSLSAs: []string{"445", "1232244"}, }, allowedEdges: edgeMap{model.EdgeBuilderHasSlsa: true}, - want: []uint32{445, 1232244}, + want: []string{"445", "1232244"}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { b := &builderStruct{ - id: tt.fields.id, - uri: tt.fields.uri, - hasSLSAs: tt.fields.hasSLSAs, + ThisID: tt.fields.id, + URI: tt.fields.uri, + HasSLSAs: tt.fields.hasSLSAs, } if got := b.Neighbors(tt.allowedEdges); !reflect.DeepEqual(got, tt.want) { t.Errorf("builderStruct.Neighbors() = %v, want %v", got, tt.want) @@ -84,7 +83,7 @@ func Test_builderStruct_Neighbors(t *testing.T) { func Test_builderStruct_BuildModelNode(t *testing.T) { type fields struct { - id uint32 + id string uri string } tests := []struct { @@ -95,37 +94,35 @@ func Test_builderStruct_BuildModelNode(t *testing.T) { }{{ name: "HubHostedActions", fields: fields{ - id: uint32(43), + id: "43", uri: "https://github.com/CreateFork/HubHostedActions@v1", }, want: &model.Builder{ - ID: nodeID(uint32(43)), + ID: "43", URI: "https://github.com/CreateFork/HubHostedActions@v1", }, wantErr: false, }, { name: "chains", fields: fields{ - id: uint32(53), + id: "53", uri: "https://tekton.dev/chains/v2", }, want: &model.Builder{ - ID: nodeID(uint32(53)), + ID: "53", URI: "https://tekton.dev/chains/v2", }, wantErr: false, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - builders: builderMap{}, - index: indexType{}, - } + c, _ := getBackend(context.Background(), nil) b := &builderStruct{ - id: tt.fields.id, - uri: tt.fields.uri, + ThisID: tt.fields.id, + URI: tt.fields.uri, } - got, err := b.BuildModelNode(c) + dc := c.(*demoClient) + got, err := b.BuildModelNode(context.Background(), dc) if (err != nil) != tt.wantErr { t.Errorf("builderStruct.BuildModelNode() error = %v, wantErr %v", err, tt.wantErr) return @@ -169,10 +166,7 @@ func Test_demoClient_IngestBuilder(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - builders: builderMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) got, err := c.IngestBuilder(ctx, tt.builderInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestBuilder() error = %v, wantErr %v", err, tt.wantErr) @@ -216,10 +210,7 @@ func Test_demoClient_IngestBuilders(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - builders: builderMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) got, err := c.IngestBuilders(ctx, tt.builderInputs) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestBuilder() error = %v, wantErr %v", err, tt.wantErr) @@ -282,10 +273,7 @@ func Test_demoClient_Builders(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - builders: builderMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) ingestedBuilder, err := c.IngestBuilder(ctx, tt.builderInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestBuilder() error = %v, wantErr %v", err, tt.wantErr) @@ -324,7 +312,7 @@ func Test_demoClient_exactBuilder(t *testing.T) { URI: ptrfrom.String("https://github.com/CreateFork/HubHostedActions@v1"), }, want: &builderStruct{ - uri: "https://github.com/CreateFork/HubHostedActions@v1", + URI: "https://github.com/CreateFork/HubHostedActions@v1", }, wantErr: false, }, { @@ -337,16 +325,13 @@ func Test_demoClient_exactBuilder(t *testing.T) { }, idInFilter: true, want: &builderStruct{ - uri: "https://tekton.dev/chains/v2", + URI: "https://tekton.dev/chains/v2", }, wantErr: false, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - builders: builderMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) ingestedBuilder, err := c.IngestBuilder(ctx, tt.builderInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestBuilder() error = %v, wantErr %v", err, tt.wantErr) @@ -355,18 +340,13 @@ func Test_demoClient_exactBuilder(t *testing.T) { if tt.idInFilter { tt.builderSpec.ID = &ingestedBuilder.ID } - got, err := c.exactBuilder(tt.builderSpec) + dc := c.(*demoClient) + got, err := dc.exactBuilder(ctx, tt.builderSpec) if (err != nil) != tt.wantErr { t.Errorf("demoClient.exactBuilder() error = %v, wantErr %v", err, tt.wantErr) return } - id64, err := strconv.ParseUint(ingestedBuilder.ID, 10, 32) - if err != nil { - t.Errorf("failed to convert string to uint32: %v", err) - return - } - id := uint32(id64) - tt.want.id = id + tt.want.ThisID = ingestedBuilder.ID if !reflect.DeepEqual(got, tt.want) { t.Errorf("demoClient.exactBuilder() = %v, want %v", got, tt.want) } diff --git a/pkg/assembler/backends/inmem/certifyBad.go b/pkg/assembler/backends/keyvalue/certifyBad.go similarity index 52% rename from pkg/assembler/backends/inmem/certifyBad.go rename to pkg/assembler/backends/keyvalue/certifyBad.go index 92924388f4..e44b8a4615 100644 --- a/pkg/assembler/backends/inmem/certifyBad.go +++ b/pkg/assembler/backends/keyvalue/certifyBad.go @@ -13,51 +13,64 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // TODO: update the other backends to handle the new timestamp fields beacuse of: https://github.com/guacsec/guac/pull/1338/files#r1343080326 // Internal data: link that a package/source/artifact is bad -type badList []*badLink type badLink struct { - id uint32 - packageID uint32 - artifactID uint32 - sourceID uint32 - justification string - origin string - collector string - knownSince time.Time + ThisID string + PackageID string + ArtifactID string + SourceID string + Justification string + Origin string + Collector string + KnownSince time.Time } -func (n *badLink) ID() uint32 { return n.id } +func (n *badLink) ID() string { return n.ThisID } -func (n *badLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 1) - if n.packageID != 0 && allowedEdges[model.EdgeCertifyBadPackage] { - out = append(out, n.packageID) +func (n *badLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.ArtifactID, + n.SourceID, + n.Justification, + n.Origin, + n.Collector, + timeKey(n.KnownSince), + }, ":") +} + +func (n *badLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 1) + if n.PackageID != "" && allowedEdges[model.EdgeCertifyBadPackage] { + out = append(out, n.PackageID) } - if n.artifactID != 0 && allowedEdges[model.EdgeCertifyBadArtifact] { - out = append(out, n.artifactID) + if n.ArtifactID != "" && allowedEdges[model.EdgeCertifyBadArtifact] { + out = append(out, n.ArtifactID) } - if n.sourceID != 0 && allowedEdges[model.EdgeCertifyBadSource] { - out = append(out, n.sourceID) + if n.SourceID != "" && allowedEdges[model.EdgeCertifyBadSource] { + out = append(out, n.SourceID) } return out } -func (n *badLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildCertifyBad(n, nil, true) +func (n *badLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildCertifyBad(ctx, n, nil, true) } // Ingest CertifyBad @@ -97,117 +110,80 @@ func (c *demoClient) IngestCertifyBad(ctx context.Context, subject model.Package func (c *demoClient) ingestCertifyBad(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, certifyBad model.CertifyBadInputSpec, readOnly bool) (*model.CertifyBad, error) { funcName := "IngestCertifyBad" + in := &badLink{ + Justification: certifyBad.Justification, + Origin: certifyBad.Origin, + Collector: certifyBad.Collector, + KnownSince: certifyBad.KnownSince.UTC(), + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var packageID uint32 var foundPkgNameorVersionNode pkgNameOrVersion - var artifactID uint32 - var foundArtStrct *artStruct - var sourceID uint32 - var srcName *srcNameNode - var searchIDs []uint32 + var foundArtStruct *artStruct + var foundSrcName *srcNameNode + if subject.Package != nil { var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, *pkgMatchType) + foundPkgNameorVersionNode, err = c.getPackageNameOrVerFromInput(ctx, *subject.Package, *pkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundPkgNameorVersionNode, err = byID[pkgNameOrVersion](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = foundPkgNameorVersionNode.getCertifyBadLinks() + in.PackageID = foundPkgNameorVersionNode.ID() } else if subject.Artifact != nil { var err error - artifactID, err = getArtifactIDFromInput(c, *subject.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundArtStrct, err = byID[*artStruct](artifactID, c) + foundArtStruct, err = c.artifactByInput(ctx, subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - searchIDs = foundArtStrct.badLinks + in.ArtifactID = foundArtStruct.ThisID } else { var err error - sourceID, err = getSourceIDFromInput(c, *subject.Source) + foundSrcName, err = c.getSourceNameFromInput(ctx, *subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - srcName, err = byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = srcName.badLinks + in.SourceID = foundSrcName.ThisID } - // Don't insert duplicates - duplicate := false - collectedCertifyBadLink := badLink{} - for _, id := range searchIDs { - v, err := byID[*badLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectMatch := false - if packageID != 0 && packageID == v.packageID { - subjectMatch = true - } - if artifactID != 0 && artifactID == v.artifactID { - subjectMatch = true - } - if sourceID != 0 && sourceID == v.sourceID { - subjectMatch = true - } - if subjectMatch && certifyBad.Justification == v.justification && - certifyBad.Origin == v.origin && certifyBad.Collector == v.collector && - certifyBad.KnownSince.Equal(v.knownSince) { + out, err := byKeykv[*badLink](ctx, cbCol, in.Key(), c) + if err == nil { + return c.buildCertifyBad(ctx, out, nil, true) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } - collectedCertifyBadLink = *v - duplicate = true - break - } + if readOnly { + c.m.RUnlock() + b, err := c.ingestCertifyBad(ctx, subject, pkgMatchType, certifyBad, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - b, err := c.ingestCertifyBad(ctx, subject, pkgMatchType, certifyBad, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - // store the link - collectedCertifyBadLink = badLink{ - id: c.getNextID(), - packageID: packageID, - artifactID: artifactID, - sourceID: sourceID, - justification: certifyBad.Justification, - origin: certifyBad.Origin, - collector: certifyBad.Collector, - knownSince: certifyBad.KnownSince.UTC(), - } - c.index[collectedCertifyBadLink.id] = &collectedCertifyBadLink - c.certifyBads = append(c.certifyBads, &collectedCertifyBadLink) - // set the backlinks - if packageID != 0 { - foundPkgNameorVersionNode.setCertifyBadLinks(collectedCertifyBadLink.id) + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, cbCol, in); err != nil { + return nil, err + } + if foundPkgNameorVersionNode != nil { + if err := foundPkgNameorVersionNode.setCertifyBadLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if artifactID != 0 { - foundArtStrct.setCertifyBadLinks(collectedCertifyBadLink.id) + } else if foundArtStruct != nil { + if err := foundArtStruct.setCertifyBadLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if sourceID != 0 { - srcName.setCertifyBadLinks(collectedCertifyBadLink.id) + } else { + if err := foundSrcName.setCertifyBadLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - } - - // build return GraphQL type - builtCertifyBad, err := c.buildCertifyBad(&collectedCertifyBadLink, nil, true) - if err != nil { + if err := setkv(ctx, cbCol, in, c); err != nil { return nil, err } - return builtCertifyBad, nil + + return c.buildCertifyBad(ctx, in, nil, true) } // Query CertifyBad @@ -218,17 +194,12 @@ func (c *demoClient) CertifyBad(ctx context.Context, filter *model.CertifyBadSpe defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*badLink](id, c) + link, err := byIDkv[*badLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - foundCertifyBad, err := c.buildCertifyBad(link, filter, true) + foundCertifyBad, err := c.buildCertifyBad(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -237,25 +208,25 @@ func (c *demoClient) CertifyBad(ctx context.Context, filter *model.CertifyBadSpe // Cant really search for an exact Pkg, as these can be linked to either // names or versions, and version could be empty. - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Subject.Artifact) + exactArtifact, err := c.artifactExact(ctx, filter.Subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.badLinks...) + search = append(search, exactArtifact.BadLinks...) foundOne = true } } if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) + exactSource, err := c.exactSource(ctx, filter.Subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.badLinks...) + search = append(search, exactSource.BadLinks...) foundOne = true } } @@ -263,19 +234,26 @@ func (c *demoClient) CertifyBad(ctx context.Context, filter *model.CertifyBadSpe var out []*model.CertifyBad if foundOne { for _, id := range search { - link, err := byID[*badLink](id, c) + link, err := byIDkv[*badLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addCBIfMatch(out, filter, link) + out, err = c.addCBIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.certifyBads { - var err error - out, err = c.addCBIfMatch(out, filter, link) + cgKeys, err := c.kv.Keys(ctx, cbCol) + if err != nil { + return nil, err + } + for _, cgk := range cgKeys { + link, err := byKeykv[*badLink](ctx, cbCol, cgk, c) + if err != nil { + return nil, err + } + out, err = c.addCBIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -284,20 +262,20 @@ func (c *demoClient) CertifyBad(ctx context.Context, filter *model.CertifyBadSpe return out, nil } -func (c *demoClient) addCBIfMatch(out []*model.CertifyBad, +func (c *demoClient) addCBIfMatch(ctx context.Context, out []*model.CertifyBad, filter *model.CertifyBadSpec, link *badLink) ( []*model.CertifyBad, error) { if filter != nil { - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Collector, link.collector) || - noMatch(filter.Origin, link.origin) || - (filter.KnownSince != nil && filter.KnownSince.After(link.knownSince)) { + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Collector, link.Collector) || + noMatch(filter.Origin, link.Origin) || + filter.KnownSince != nil && filter.KnownSince.After(link.KnownSince) { return out, nil } } - foundCertifyBad, err := c.buildCertifyBad(link, filter, false) + foundCertifyBad, err := c.buildCertifyBad(ctx, link, filter, false) if err != nil { return nil, err } @@ -307,45 +285,45 @@ func (c *demoClient) addCBIfMatch(out []*model.CertifyBad, return append(out, foundCertifyBad), nil } -func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec, ingestOrIDProvided bool) (*model.CertifyBad, error) { +func (c *demoClient) buildCertifyBad(ctx context.Context, link *badLink, filter *model.CertifyBadSpec, ingestOrIDProvided bool) (*model.CertifyBad, error) { var p *model.Package var a *model.Artifact var s *model.Source var err error if filter != nil && filter.Subject != nil { - if filter.Subject.Package != nil && link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, filter.Subject.Package) + if filter.Subject.Package != nil && link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Subject.Package) if err != nil { return nil, err } } - if filter.Subject.Artifact != nil && link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, filter.Subject.Artifact) + if filter.Subject.Artifact != nil && link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, filter.Subject.Artifact) if err != nil { return nil, err } } - if filter.Subject.Source != nil && link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, filter.Subject.Source) + if filter.Subject.Source != nil && link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Subject.Source) if err != nil { return nil, err } } } else { - if link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, nil) + if link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } } - if link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, nil) + if link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, nil) if err != nil { return nil, err } } - if link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, nil) + if link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -353,7 +331,7 @@ func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec } var subj model.PackageSourceOrArtifact - if link.packageID != 0 { + if link.PackageID != "" { if p == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve package via packageID") } else if p == nil && !ingestOrIDProvided { @@ -361,7 +339,7 @@ func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec } subj = p } - if link.artifactID != 0 { + if link.ArtifactID != "" { if a == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") } else if a == nil && !ingestOrIDProvided { @@ -369,7 +347,7 @@ func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec } subj = a } - if link.sourceID != 0 { + if link.SourceID != "" { if s == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve source via sourceID") } else if s == nil && !ingestOrIDProvided { @@ -379,12 +357,12 @@ func (c *demoClient) buildCertifyBad(link *badLink, filter *model.CertifyBadSpec } certifyBad := model.CertifyBad{ - ID: nodeID(link.id), + ID: link.ThisID, Subject: subj, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, - KnownSince: link.knownSince.UTC(), + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, + KnownSince: link.KnownSince.UTC(), } return &certifyBad, nil } diff --git a/pkg/assembler/backends/inmem/certifyBad_test.go b/pkg/assembler/backends/keyvalue/certifyBad_test.go similarity index 97% rename from pkg/assembler/backends/inmem/certifyBad_test.go rename to pkg/assembler/backends/keyvalue/certifyBad_test.go index ede819a2af..f47f55e141 100644 --- a/pkg/assembler/backends/inmem/certifyBad_test.go +++ b/pkg/assembler/backends/keyvalue/certifyBad_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -463,11 +464,11 @@ func TestCertifyBad(t *testing.T) { }, ExpCB: []*model.CertifyBad{ { - Subject: p2out, + Subject: p1outName, Justification: "test justification", }, { - Subject: p1outName, + Subject: p2out, Justification: "test justification", }, }, @@ -517,24 +518,6 @@ func TestCertifyBad(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Sub: model.PackageSourceOrArtifactInput{ - Source: s1, - }, - CB: &model.CertifyBadInputSpec{ - Justification: "test justification", - }, - }, - }, - Query: &model.CertifyBadSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -542,7 +525,8 @@ func TestCertifyBad(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -830,7 +814,8 @@ func TestIngestCertifyBads(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -959,7 +944,8 @@ func TestCertifyBadNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/certifyGood.go b/pkg/assembler/backends/keyvalue/certifyGood.go similarity index 51% rename from pkg/assembler/backends/inmem/certifyGood.go rename to pkg/assembler/backends/keyvalue/certifyGood.go index 1d8b93a84e..97dc8305da 100644 --- a/pkg/assembler/backends/inmem/certifyGood.go +++ b/pkg/assembler/backends/keyvalue/certifyGood.go @@ -13,49 +13,62 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link that a package/source/artifact is good -type goodList []*goodLink type goodLink struct { - id uint32 - packageID uint32 - artifactID uint32 - sourceID uint32 - justification string - origin string - collector string - knownSince time.Time + ThisID string + PackageID string + ArtifactID string + SourceID string + Justification string + Origin string + Collector string + KnownSince time.Time } -func (n *goodLink) ID() uint32 { return n.id } +func (n *goodLink) ID() string { return n.ThisID } -func (n *goodLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 1) - if n.packageID != 0 && allowedEdges[model.EdgeCertifyGoodPackage] { - out = append(out, n.packageID) +func (n *goodLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.ArtifactID, + n.SourceID, + n.Justification, + n.Origin, + n.Collector, + timeKey(n.KnownSince), + }, ":") +} + +func (n *goodLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 1) + if n.PackageID != "" && allowedEdges[model.EdgeCertifyGoodPackage] { + out = append(out, n.PackageID) } - if n.artifactID != 0 && allowedEdges[model.EdgeCertifyGoodArtifact] { - out = append(out, n.artifactID) + if n.ArtifactID != "" && allowedEdges[model.EdgeCertifyGoodArtifact] { + out = append(out, n.ArtifactID) } - if n.sourceID != 0 && allowedEdges[model.EdgeCertifyGoodSource] { - out = append(out, n.sourceID) + if n.SourceID != "" && allowedEdges[model.EdgeCertifyGoodSource] { + out = append(out, n.SourceID) } return out } -func (n *goodLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildCertifyGood(n, nil, true) +func (n *goodLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildCertifyGood(ctx, n, nil, true) } // Ingest CertifyGood @@ -96,117 +109,80 @@ func (c *demoClient) IngestCertifyGood(ctx context.Context, subject model.Packag func (c *demoClient) ingestCertifyGood(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, certifyGood model.CertifyGoodInputSpec, readOnly bool) (*model.CertifyGood, error) { funcName := "IngestCertifyGood" + in := &goodLink{ + Justification: certifyGood.Justification, + Origin: certifyGood.Origin, + Collector: certifyGood.Collector, + KnownSince: certifyGood.KnownSince.UTC(), + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var packageID uint32 var foundPkgNameorVersionNode pkgNameOrVersion - var artifactID uint32 var foundArtStrct *artStruct - var sourceID uint32 - var srcName *srcNameNode - searchIDs := []uint32{} + var foundSrcName *srcNameNode + if subject.Package != nil { var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, *pkgMatchType) + foundPkgNameorVersionNode, err = c.getPackageNameOrVerFromInput(ctx, *subject.Package, *pkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundPkgNameorVersionNode, err = byID[pkgNameOrVersion](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = append(searchIDs, foundPkgNameorVersionNode.getCertifyGoodLinks()...) + in.PackageID = foundPkgNameorVersionNode.ID() } else if subject.Artifact != nil { var err error - artifactID, err = getArtifactIDFromInput(c, *subject.Artifact) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundArtStrct, err = byID[*artStruct](artifactID, c) + foundArtStrct, err = c.artifactByInput(ctx, subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - searchIDs = append(searchIDs, foundArtStrct.goodLinks...) + in.ArtifactID = foundArtStrct.ThisID } else { var err error - sourceID, err = getSourceIDFromInput(c, *subject.Source) + foundSrcName, err = c.getSourceNameFromInput(ctx, *subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - srcName, err = byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = append(searchIDs, srcName.goodLinks...) + in.SourceID = foundSrcName.ThisID } - // Don't insert duplicates - duplicate := false - collectedCertifyGoodLink := goodLink{} - for _, id := range searchIDs { - v, err := byID[*goodLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectMatch := false - if packageID != 0 && packageID == v.packageID { - subjectMatch = true - } - if artifactID != 0 && artifactID == v.artifactID { - subjectMatch = true - } - if sourceID != 0 && sourceID == v.sourceID { - subjectMatch = true - } - if subjectMatch && certifyGood.Justification == v.justification && - certifyGood.Origin == v.origin && certifyGood.Collector == v.collector && - certifyGood.KnownSince.Equal(v.knownSince) { + out, err := byKeykv[*goodLink](ctx, cgCol, in.Key(), c) + if err == nil { + return c.buildCertifyGood(ctx, out, nil, true) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } - collectedCertifyGoodLink = *v - duplicate = true - break - } + if readOnly { + c.m.RUnlock() + b, err := c.ingestCertifyGood(ctx, subject, pkgMatchType, certifyGood, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - b, err := c.ingestCertifyGood(ctx, subject, pkgMatchType, certifyGood, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - // store the link - collectedCertifyGoodLink = goodLink{ - id: c.getNextID(), - packageID: packageID, - artifactID: artifactID, - sourceID: sourceID, - justification: certifyGood.Justification, - origin: certifyGood.Origin, - collector: certifyGood.Collector, - knownSince: certifyGood.KnownSince.UTC(), - } - c.index[collectedCertifyGoodLink.id] = &collectedCertifyGoodLink - c.certifyGoods = append(c.certifyGoods, &collectedCertifyGoodLink) - // set the backlinks - if packageID != 0 { - foundPkgNameorVersionNode.setCertifyGoodLinks(collectedCertifyGoodLink.id) + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, cgCol, in); err != nil { + return nil, err + } + if foundPkgNameorVersionNode != nil { + if err := foundPkgNameorVersionNode.setCertifyGoodLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if artifactID != 0 { - foundArtStrct.setCertifyGoodLinks(collectedCertifyGoodLink.id) + } else if foundArtStrct != nil { + if err := foundArtStrct.setCertifyGoodLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if sourceID != 0 { - srcName.setCertifyGoodLinks(collectedCertifyGoodLink.id) + } else { + if err := foundSrcName.setCertifyGoodLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - } - - // build return GraphQL type - builtCertifyGood, err := c.buildCertifyGood(&collectedCertifyGoodLink, nil, true) - if err != nil { + if err := setkv(ctx, cgCol, in, c); err != nil { return nil, err } - return builtCertifyGood, nil + + return c.buildCertifyGood(ctx, in, nil, true) } // Query CertifyGood @@ -217,17 +193,12 @@ func (c *demoClient) CertifyGood(ctx context.Context, filter *model.CertifyGoodS defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*goodLink](id, c) + link, err := byIDkv[*goodLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - foundCertifyGood, err := c.buildCertifyGood(link, filter, true) + foundCertifyGood, err := c.buildCertifyGood(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -236,25 +207,25 @@ func (c *demoClient) CertifyGood(ctx context.Context, filter *model.CertifyGoodS // Cant really search for an exact Pkg, as these can be linked to either // names or versions, and version could be empty. - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Subject.Artifact) + exactArtifact, err := c.artifactExact(ctx, filter.Subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.goodLinks...) + search = append(search, exactArtifact.GoodLinks...) foundOne = true } } if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) + exactSource, err := c.exactSource(ctx, filter.Subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.goodLinks...) + search = append(search, exactSource.GoodLinks...) foundOne = true } } @@ -262,19 +233,26 @@ func (c *demoClient) CertifyGood(ctx context.Context, filter *model.CertifyGoodS var out []*model.CertifyGood if foundOne { for _, id := range search { - link, err := byID[*goodLink](id, c) + link, err := byIDkv[*goodLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addCGIfMatch(out, filter, link) + out, err = c.addCGIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.certifyGoods { - var err error - out, err = c.addCGIfMatch(out, filter, link) + cgKeys, err := c.kv.Keys(ctx, cgCol) + if err != nil { + return nil, err + } + for _, cgk := range cgKeys { + link, err := byKeykv[*goodLink](ctx, cgCol, cgk, c) + if err != nil { + return nil, err + } + out, err = c.addCGIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -283,21 +261,20 @@ func (c *demoClient) CertifyGood(ctx context.Context, filter *model.CertifyGoodS return out, nil } -func (c *demoClient) addCGIfMatch(out []*model.CertifyGood, +func (c *demoClient) addCGIfMatch(ctx context.Context, out []*model.CertifyGood, filter *model.CertifyGoodSpec, link *goodLink) ( []*model.CertifyGood, error) { if filter != nil { - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Collector, link.collector) || - noMatch(filter.Collector, link.collector) || - noMatch(filter.Origin, link.origin) || - filter.KnownSince != nil && filter.KnownSince.After(link.knownSince) { + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Collector, link.Collector) || + noMatch(filter.Origin, link.Origin) || + filter.KnownSince != nil && filter.KnownSince.After(link.KnownSince) { return out, nil } } - foundCertifyGood, err := c.buildCertifyGood(link, filter, false) + foundCertifyGood, err := c.buildCertifyGood(ctx, link, filter, false) if err != nil { return nil, err } @@ -307,45 +284,45 @@ func (c *demoClient) addCGIfMatch(out []*model.CertifyGood, return append(out, foundCertifyGood), nil } -func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodSpec, ingestOrIDProvided bool) (*model.CertifyGood, error) { +func (c *demoClient) buildCertifyGood(ctx context.Context, link *goodLink, filter *model.CertifyGoodSpec, ingestOrIDProvided bool) (*model.CertifyGood, error) { var p *model.Package var a *model.Artifact var s *model.Source var err error if filter != nil && filter.Subject != nil { - if filter.Subject.Package != nil && link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, filter.Subject.Package) + if filter.Subject.Package != nil && link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Subject.Package) if err != nil { return nil, err } } - if filter.Subject.Artifact != nil && link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, filter.Subject.Artifact) + if filter.Subject.Artifact != nil && link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, filter.Subject.Artifact) if err != nil { return nil, err } } - if filter.Subject.Source != nil && link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, filter.Subject.Source) + if filter.Subject.Source != nil && link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Subject.Source) if err != nil { return nil, err } } } else { - if link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, nil) + if link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } } - if link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, nil) + if link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, nil) if err != nil { return nil, err } } - if link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, nil) + if link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -353,7 +330,7 @@ func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodS } var subj model.PackageSourceOrArtifact - if link.packageID != 0 { + if link.PackageID != "" { if p == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve package via packageID") } else if p == nil && !ingestOrIDProvided { @@ -361,7 +338,7 @@ func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodS } subj = p } - if link.artifactID != 0 { + if link.ArtifactID != "" { if a == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") } else if a == nil && !ingestOrIDProvided { @@ -369,7 +346,7 @@ func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodS } subj = a } - if link.sourceID != 0 { + if link.SourceID != "" { if s == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve source via sourceID") } else if s == nil && !ingestOrIDProvided { @@ -379,12 +356,12 @@ func (c *demoClient) buildCertifyGood(link *goodLink, filter *model.CertifyGoodS } certifyGood := model.CertifyGood{ - ID: nodeID(link.id), + ID: link.ThisID, Subject: subj, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, - KnownSince: link.knownSince.UTC(), + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, + KnownSince: link.KnownSince.UTC(), } return &certifyGood, nil } diff --git a/pkg/assembler/backends/inmem/certifyGood_test.go b/pkg/assembler/backends/keyvalue/certifyGood_test.go similarity index 97% rename from pkg/assembler/backends/inmem/certifyGood_test.go rename to pkg/assembler/backends/keyvalue/certifyGood_test.go index 04879f2b97..3df67800d4 100644 --- a/pkg/assembler/backends/inmem/certifyGood_test.go +++ b/pkg/assembler/backends/keyvalue/certifyGood_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -463,11 +464,11 @@ func TestCertifyGood(t *testing.T) { }, ExpCG: []*model.CertifyGood{ { - Subject: p2out, + Subject: p1outName, Justification: "test justification", }, { - Subject: p1outName, + Subject: p2out, Justification: "test justification", }, }, @@ -517,24 +518,6 @@ func TestCertifyGood(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query good ID", - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Sub: model.PackageSourceOrArtifactInput{ - Source: s1, - }, - CG: &model.CertifyGoodInputSpec{ - Justification: "test justification", - }, - }, - }, - Query: &model.CertifyGoodSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -542,7 +525,8 @@ func TestCertifyGood(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -830,7 +814,8 @@ func TestIngestCertifyGoods(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -959,7 +944,8 @@ func TestCertifyGoodNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/certifyLegal.go b/pkg/assembler/backends/keyvalue/certifyLegal.go similarity index 52% rename from pkg/assembler/backends/inmem/certifyLegal.go rename to pkg/assembler/backends/keyvalue/certifyLegal.go index 59d30d3b76..ab1dc20782 100644 --- a/pkg/assembler/backends/inmem/certifyLegal.go +++ b/pkg/assembler/backends/keyvalue/certifyLegal.go @@ -13,57 +13,72 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" + "fmt" "slices" - "strconv" + "strings" "time" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) // Internal certifyLegal -type ( - certifyLegalList []*certifyLegalStruct - certifyLegalStruct struct { - id uint32 - pkg uint32 - source uint32 - declaredLicense string - declaredLicenses []uint32 - discoveredLicense string - discoveredLicenses []uint32 - attribution string - justification string - timeScanned time.Time - origin string - collector string - } -) +type certifyLegalStruct struct { + ThisID string + Pkg string + Source string + DeclaredLicense string + DeclaredLicenses []string + DiscoveredLicense string + DiscoveredLicenses []string + Attribution string + Justification string + TimeScanned time.Time + Origin string + Collector string +} -func (n *certifyLegalStruct) ID() uint32 { return n.id } +func (n *certifyLegalStruct) ID() string { return n.ThisID } +func (n *certifyLegalStruct) Key() string { + return strings.Join([]string{ + n.Pkg, + n.Source, + n.DeclaredLicense, + fmt.Sprint(n.DeclaredLicenses), + n.DiscoveredLicense, + fmt.Sprint(n.DiscoveredLicenses), + n.Attribution, + n.Justification, + timeKey(n.TimeScanned), + n.Origin, + n.Collector, + }, ":") +} -func (n *certifyLegalStruct) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2) - if n.pkg != 0 && allowedEdges[model.EdgeCertifyLegalPackage] { - out = append(out, n.pkg) +func (n *certifyLegalStruct) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 2) + if n.Pkg != "" && allowedEdges[model.EdgeCertifyLegalPackage] { + out = append(out, n.Pkg) } - if n.source != 0 && allowedEdges[model.EdgeCertifyLegalSource] { - out = append(out, n.source) + if n.Source != "" && allowedEdges[model.EdgeCertifyLegalSource] { + out = append(out, n.Source) } if allowedEdges[model.EdgeCertifyLegalLicense] { - out = append(out, n.declaredLicenses...) - out = append(out, n.discoveredLicenses...) + out = append(out, n.DeclaredLicenses...) + out = append(out, n.DiscoveredLicenses...) } return out } -func (n *certifyLegalStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convLegal(n) +func (n *certifyLegalStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convLegal(ctx, n) } func (c *demoClient) IngestCertifyLegals(ctx context.Context, subjects model.PackageOrSourceInputs, declaredLicensesList [][]*model.LicenseInputSpec, discoveredLicensesList [][]*model.LicenseInputSpec, certifyLegals []*model.CertifyLegalInputSpec) ([]*model.CertifyLegal, error) { @@ -97,158 +112,147 @@ func (c *demoClient) IngestCertifyLegal(ctx context.Context, subject model.Packa func (c *demoClient) ingestCertifyLegal(ctx context.Context, subject model.PackageOrSourceInput, declaredLicenses []*model.LicenseInputSpec, discoveredLicenses []*model.LicenseInputSpec, certifyLegal *model.CertifyLegalInputSpec, readOnly bool) (*model.CertifyLegal, error) { funcName := "IngestCertifyLegal" + in := &certifyLegalStruct{ + DeclaredLicense: certifyLegal.DeclaredLicense, + DiscoveredLicense: certifyLegal.DiscoveredLicense, + Attribution: certifyLegal.Attribution, + TimeScanned: certifyLegal.TimeScanned.UTC(), + Justification: certifyLegal.Justification, + Origin: certifyLegal.Origin, + Collector: certifyLegal.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var dec []uint32 + var dec []string for _, lis := range declaredLicenses { - l, ok := c.licenses[licenseKey(lis.Name, lis.ListVersion)] - if !ok { - return nil, gqlerror.Errorf("%v :: License not found %q", funcName, licenseKey(lis.Name, lis.ListVersion)) + l, err := c.licenseByInput(ctx, lis) + if err != nil { + return nil, gqlerror.Errorf("%v :: License not found %q %v", funcName, lis.Name, err) } - dec = append(dec, l.id) + dec = append(dec, l.ThisID) } slices.Sort(dec) - var dis []uint32 + in.DeclaredLicenses = dec + + var dis []string for _, lis := range discoveredLicenses { - l, ok := c.licenses[licenseKey(lis.Name, lis.ListVersion)] - if !ok { - return nil, gqlerror.Errorf("%v :: License not found %q", funcName, licenseKey(lis.Name, lis.ListVersion)) + l, err := c.licenseByInput(ctx, lis) + if err != nil { + return nil, gqlerror.Errorf("%v :: License not found %q %v", funcName, lis.Name, err) } - dis = append(dis, l.id) + dis = append(dis, l.ThisID) } slices.Sort(dis) + in.DiscoveredLicenses = dis - var backedgeSearch []uint32 - var packageID uint32 - var pkg *pkgVersionNode + var pkg *pkgVersion if subject.Package != nil { - var pmt model.MatchFlags - pmt.Pkg = model.PkgMatchTypeSpecificVersion - pid, err := getPackageIDFromInput(c, *subject.Package, pmt) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - packageID = pid - pkg, err = byID[*pkgVersionNode](packageID, c) + var err error + pkg, err = c.getPackageVerFromInput(ctx, *subject.Package) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - backedgeSearch = pkg.certifyLegals + in.Pkg = pkg.ThisID } - var sourceID uint32 var src *srcNameNode if subject.Source != nil { - sid, err := getSourceIDFromInput(c, *subject.Source) + var err error + src, err = c.getSourceNameFromInput(ctx, *subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - sourceID = sid - src, err = byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - backedgeSearch = src.certifyLegals + in.Source = src.ThisID } - for _, id := range backedgeSearch { - cl, err := byID[*certifyLegalStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if cl.pkg == packageID && - cl.source == sourceID && - cl.declaredLicense == certifyLegal.DeclaredLicense && - slices.Equal(cl.declaredLicenses, dec) && - cl.discoveredLicense == certifyLegal.DiscoveredLicense && - slices.Equal(cl.discoveredLicenses, dis) && - cl.attribution == certifyLegal.Attribution && - cl.timeScanned.Equal(certifyLegal.TimeScanned) && - cl.justification == certifyLegal.Justification && - cl.origin == certifyLegal.Origin && - cl.collector == certifyLegal.Collector { - return c.convLegal(cl) - } + out, err := byKeykv[*certifyLegalStruct](ctx, clCol, in.Key(), c) + if err == nil { + return c.convLegal(ctx, out) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } + if readOnly { c.m.RUnlock() o, err := c.ingestCertifyLegal(ctx, subject, declaredLicenses, discoveredLicenses, certifyLegal, false) c.m.RLock() // relock so that defer unlock does not panic return o, err } - cl := &certifyLegalStruct{ - id: c.getNextID(), - pkg: packageID, - source: sourceID, - declaredLicense: certifyLegal.DeclaredLicense, - declaredLicenses: dec, - discoveredLicense: certifyLegal.DiscoveredLicense, - discoveredLicenses: dis, - attribution: certifyLegal.Attribution, - timeScanned: certifyLegal.TimeScanned, - justification: certifyLegal.Justification, - origin: certifyLegal.Origin, - collector: certifyLegal.Collector, - } - c.index[cl.id] = cl - if packageID != 0 { - pkg.setCertifyLegals(cl.id) + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, clCol, in); err != nil { + return nil, err + } + if pkg != nil { + if err := pkg.setCertifyLegals(ctx, in.ThisID, c); err != nil { + return nil, err + } } else { - src.setCertifyLegals(cl.id) + if err := src.setCertifyLegals(ctx, in.ThisID, c); err != nil { + return nil, err + } } for _, lid := range dec { - l, err := byID[*licStruct](lid, c) + l, err := byIDkv[*licStruct](ctx, lid, c) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - l.setCertifyLegals(cl.id) + if err := l.setCertifyLegals(ctx, in.ThisID, c); err != nil { + return nil, err + } } for _, lid := range dis { - l, err := byID[*licStruct](lid, c) + l, err := byIDkv[*licStruct](ctx, lid, c) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - l.setCertifyLegals(cl.id) + if err := l.setCertifyLegals(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := setkv(ctx, clCol, in, c); err != nil { + return nil, err } - c.certifyLegals = append(c.certifyLegals, cl) - return c.convLegal(cl) + return c.convLegal(ctx, in) } -func (c *demoClient) convLegal(in *certifyLegalStruct) (*model.CertifyLegal, error) { +func (c *demoClient) convLegal(ctx context.Context, in *certifyLegalStruct) (*model.CertifyLegal, error) { cl := &model.CertifyLegal{ - ID: nodeID(in.id), - DeclaredLicense: in.declaredLicense, - DiscoveredLicense: in.discoveredLicense, - Attribution: in.attribution, - Justification: in.justification, - TimeScanned: in.timeScanned, - Origin: in.origin, - Collector: in.collector, - } - for _, lid := range in.declaredLicenses { - l, err := byID[*licStruct](lid, c) + ID: in.ThisID, + DeclaredLicense: in.DeclaredLicense, + DiscoveredLicense: in.DiscoveredLicense, + Attribution: in.Attribution, + Justification: in.Justification, + TimeScanned: in.TimeScanned, + Origin: in.Origin, + Collector: in.Collector, + } + for _, lid := range in.DeclaredLicenses { + l, err := byIDkv[*licStruct](ctx, lid, c) if err != nil { return nil, err } cl.DeclaredLicenses = append(cl.DeclaredLicenses, c.convLicense(l)) } - for _, lid := range in.discoveredLicenses { - l, err := byID[*licStruct](lid, c) + for _, lid := range in.DiscoveredLicenses { + l, err := byIDkv[*licStruct](ctx, lid, c) if err != nil { return nil, err } cl.DiscoveredLicenses = append(cl.DiscoveredLicenses, c.convLicense(l)) } - if in.pkg != 0 { - p, err := c.buildPackageResponse(in.pkg, nil) + if in.Pkg != "" { + p, err := c.buildPackageResponse(ctx, in.Pkg, nil) if err != nil { return nil, err } cl.Subject = p } else { - s, err := c.buildSourceResponse(in.source, nil) + s, err := c.buildSourceResponse(ctx, in.Source, nil) if err != nil { return nil, err } @@ -264,54 +268,49 @@ func (c *demoClient) CertifyLegal(ctx context.Context, filter *model.CertifyLega defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*certifyLegalStruct](id, c) + link, err := byIDkv[*certifyLegalStruct](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - o, err := c.convLegal(link) + o, err := c.convLegal(ctx, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.CertifyLegal{o}, nil } - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Subject != nil && filter.Subject.Package != nil { - pkgs, err := c.findPackageVersion(filter.Subject.Package) + pkgs, err := c.findPackageVersion(ctx, filter.Subject.Package) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } foundOne = len(pkgs) > 0 for _, pkg := range pkgs { - search = append(search, pkg.certifyLegals...) + search = append(search, pkg.CertifyLegals...) } } if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) + exactSource, err := c.exactSource(ctx, filter.Subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.certifyLegals...) + search = append(search, exactSource.CertifyLegals...) foundOne = true } } if !foundOne && filter != nil { for _, lSpec := range filter.DeclaredLicenses { - exactLicense, err := c.licenseExact(lSpec) + exactLicense, err := c.licenseExact(ctx, lSpec) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactLicense != nil { - search = append(search, exactLicense.certifyLegals...) + search = append(search, exactLicense.CertifyLegals...) foundOne = true break } @@ -319,12 +318,12 @@ func (c *demoClient) CertifyLegal(ctx context.Context, filter *model.CertifyLega } if !foundOne && filter != nil { for _, lSpec := range filter.DiscoveredLicenses { - exactLicense, err := c.licenseExact(lSpec) + exactLicense, err := c.licenseExact(ctx, lSpec) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactLicense != nil { - search = append(search, exactLicense.certifyLegals...) + search = append(search, exactLicense.CertifyLegals...) foundOne = true break } @@ -334,19 +333,26 @@ func (c *demoClient) CertifyLegal(ctx context.Context, filter *model.CertifyLega var out []*model.CertifyLegal if foundOne { for _, id := range search { - link, err := byID[*certifyLegalStruct](id, c) + link, err := byIDkv[*certifyLegalStruct](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addLegalIfMatch(out, filter, link) + out, err = c.addLegalIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.certifyLegals { - var err error - out, err = c.addLegalIfMatch(out, filter, link) + clKeys, err := c.kv.Keys(ctx, clCol) + if err != nil { + return nil, err + } + for _, clk := range clKeys { + link, err := byKeykv[*certifyLegalStruct](ctx, clCol, clk, c) + if err != nil { + return nil, err + } + out, err = c.addLegalIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -355,27 +361,27 @@ func (c *demoClient) CertifyLegal(ctx context.Context, filter *model.CertifyLega return out, nil } -func (c *demoClient) addLegalIfMatch(out []*model.CertifyLegal, +func (c *demoClient) addLegalIfMatch(ctx context.Context, out []*model.CertifyLegal, filter *model.CertifyLegalSpec, link *certifyLegalStruct) ( []*model.CertifyLegal, error, ) { - if noMatch(filter.DeclaredLicense, link.declaredLicense) || - noMatch(filter.DiscoveredLicense, link.discoveredLicense) || - noMatch(filter.Attribution, link.attribution) || - noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) || - (filter.TimeScanned != nil && !link.timeScanned.Equal(*filter.TimeScanned)) || - !c.matchLicenses(filter.DeclaredLicenses, link.declaredLicenses) || - !c.matchLicenses(filter.DiscoveredLicenses, link.discoveredLicenses) { + if noMatch(filter.DeclaredLicense, link.DeclaredLicense) || + noMatch(filter.DiscoveredLicense, link.DiscoveredLicense) || + noMatch(filter.Attribution, link.Attribution) || + noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) || + (filter.TimeScanned != nil && !link.TimeScanned.Equal(*filter.TimeScanned)) || + !c.matchLicenses(ctx, filter.DeclaredLicenses, link.DeclaredLicenses) || + !c.matchLicenses(ctx, filter.DiscoveredLicenses, link.DiscoveredLicenses) { return out, nil } if filter.Subject != nil { if filter.Subject.Package != nil { - if link.pkg == 0 { + if link.Pkg == "" { return out, nil } - p, err := c.buildPackageResponse(link.pkg, filter.Subject.Package) + p, err := c.buildPackageResponse(ctx, link.Pkg, filter.Subject.Package) if err != nil { return nil, err } @@ -383,10 +389,10 @@ func (c *demoClient) addLegalIfMatch(out []*model.CertifyLegal, return out, nil } } else if filter.Subject.Source != nil { - if link.source == 0 { + if link.Source == "" { return out, nil } - s, err := c.buildSourceResponse(link.source, filter.Subject.Source) + s, err := c.buildSourceResponse(ctx, link.Source, filter.Subject.Source) if err != nil { return nil, err } @@ -395,25 +401,25 @@ func (c *demoClient) addLegalIfMatch(out []*model.CertifyLegal, } } } - o, err := c.convLegal(link) + o, err := c.convLegal(ctx, link) if err != nil { return nil, err } return append(out, o), nil } -func (c *demoClient) matchLicenses(filter []*model.LicenseSpec, value []uint32) bool { +func (c *demoClient) matchLicenses(ctx context.Context, filter []*model.LicenseSpec, value []string) bool { val := slices.Clone(value) - var matchID []uint32 + var matchID []string var matchPartial []*model.LicenseSpec for _, aSpec := range filter { if aSpec == nil { continue } - a, _ := c.licenseExact(aSpec) + a, _ := c.licenseExact(ctx, aSpec) // drop error here if ID is bad if a != nil { - matchID = append(matchID, a.id) + matchID = append(matchID, a.ThisID) } else { matchPartial = append(matchPartial, aSpec) } @@ -428,13 +434,13 @@ func (c *demoClient) matchLicenses(filter []*model.LicenseSpec, value []uint32) match := false remove := -1 for i, v := range val { - a, err := byID[*licStruct](v, c) + a, err := byIDkv[*licStruct](ctx, v, c) if err != nil { return false } - if (m.Name == nil || *m.Name == a.name) && - (m.ListVersion == nil || *m.ListVersion == a.listVersion) && - (m.Inline == nil || *m.Inline == a.inline) { + if (m.Name == nil || *m.Name == a.Name) && + (m.ListVersion == nil || *m.ListVersion == a.ListVersion) && + (m.Inline == nil || *m.Inline == a.Inline) { match = true remove = i break diff --git a/pkg/assembler/backends/inmem/certifyLegal_test.go b/pkg/assembler/backends/keyvalue/certifyLegal_test.go similarity index 98% rename from pkg/assembler/backends/inmem/certifyLegal_test.go rename to pkg/assembler/backends/keyvalue/certifyLegal_test.go index 4be26cdc58..c97d539b34 100644 --- a/pkg/assembler/backends/inmem/certifyLegal_test.go +++ b/pkg/assembler/backends/keyvalue/certifyLegal_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -486,7 +487,8 @@ func TestLegal(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -586,7 +588,8 @@ func TestLegals(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -705,7 +708,8 @@ func TestLegalNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/certifyScorecard.go b/pkg/assembler/backends/keyvalue/certifyScorecard.go similarity index 55% rename from pkg/assembler/backends/inmem/certifyScorecard.go rename to pkg/assembler/backends/keyvalue/certifyScorecard.go index 853c5f6190..559246eb00 100644 --- a/pkg/assembler/backends/inmem/certifyScorecard.go +++ b/pkg/assembler/backends/keyvalue/certifyScorecard.go @@ -13,44 +13,58 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" + "fmt" "reflect" - "strconv" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link between source and scorecard (certifyScorecard) -type scorecardList []*scorecardLink type scorecardLink struct { - id uint32 - sourceID uint32 - timeScanned time.Time - aggregateScore float64 - checks map[string]int - scorecardVersion string - scorecardCommit string - origin string - collector string + ThisID string + SourceID string + TimeScanned time.Time + AggregateScore float64 + Checks map[string]int + ScorecardVersion string + ScorecardCommit string + Origin string + Collector string } -func (n *scorecardLink) ID() uint32 { return n.id } +func (n *scorecardLink) ID() string { return n.ThisID } +func (n *scorecardLink) Key() string { + return strings.Join([]string{ + n.SourceID, + timeKey(n.TimeScanned), + fmt.Sprint(n.AggregateScore), + fmt.Sprint(n.Checks), + n.ScorecardVersion, + n.ScorecardCommit, + n.Origin, + n.Collector, + }, ":") +} -func (n *scorecardLink) Neighbors(allowedEdges edgeMap) []uint32 { +func (n *scorecardLink) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgeCertifyScorecardSource] { - return []uint32{n.sourceID} + return []string{n.SourceID} } - return []uint32{} + return nil } -func (n *scorecardLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildScorecard(n, nil, true) +func (n *scorecardLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildScorecard(ctx, n, nil, true) } // Ingest Scorecards @@ -75,74 +89,53 @@ func (c *demoClient) IngestScorecard(ctx context.Context, source model.SourceInp func (c *demoClient) certifyScorecard(ctx context.Context, source model.SourceInputSpec, scorecard model.ScorecardInputSpec, readOnly bool) (*model.CertifyScorecard, error) { funcName := "CertifyScorecard" + checksMap := getChecksFromInput(scorecard.Checks) + in := &scorecardLink{ + TimeScanned: scorecard.TimeScanned.UTC(), + AggregateScore: scorecard.AggregateScore, + Checks: checksMap, + ScorecardVersion: scorecard.ScorecardVersion, + ScorecardCommit: scorecard.ScorecardCommit, + Origin: scorecard.Origin, + Collector: scorecard.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - sourceID, err := getSourceIDFromInput(c, source) + srcName, err := c.getSourceNameFromInput(ctx, source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - srcName, err := byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + in.SourceID = srcName.ThisID + + out, err := byKeykv[*scorecardLink](ctx, cscCol, in.Key(), c) + if err == nil { + return c.buildScorecard(ctx, out, nil, true) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } - searchIDs := srcName.scorecardLinks - checksMap := getChecksFromInput(scorecard.Checks) + if readOnly { + c.m.RUnlock() + s, err := c.certifyScorecard(ctx, source, scorecard, false) + c.m.RLock() // relock so that defer unlock does not panic + return s, err + } - // Don't insert duplicates - duplicate := false - collectedScorecardLink := scorecardLink{} - for _, id := range searchIDs { - v, err := byID[*scorecardLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if sourceID == v.sourceID && - scorecard.TimeScanned.Equal(v.timeScanned) && - floatEqual(scorecard.AggregateScore, v.aggregateScore) && - scorecard.ScorecardVersion == v.scorecardVersion && - scorecard.ScorecardCommit == v.scorecardCommit && - scorecard.Origin == v.origin && - scorecard.Collector == v.collector && - reflect.DeepEqual(checksMap, v.checks) { - collectedScorecardLink = *v - duplicate = true - break - } + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, cscCol, in); err != nil { + return nil, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - s, err := c.certifyScorecard(ctx, source, scorecard, false) - c.m.RLock() // relock so that defer unlock does not panic - return s, err - } - // store the link - collectedScorecardLink = scorecardLink{ - id: c.getNextID(), - sourceID: sourceID, - timeScanned: scorecard.TimeScanned.UTC(), - aggregateScore: scorecard.AggregateScore, - checks: checksMap, - scorecardVersion: scorecard.ScorecardVersion, - scorecardCommit: scorecard.ScorecardCommit, - origin: scorecard.Origin, - collector: scorecard.Collector, - } - c.index[collectedScorecardLink.id] = &collectedScorecardLink - c.scorecards = append(c.scorecards, &collectedScorecardLink) - // set the backlinks - srcName.setScorecardLinks(collectedScorecardLink.id) + if err := srcName.setScorecardLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - - // build return GraphQL type - builtCertifyScorecard, err := c.buildScorecard(&collectedScorecardLink, nil, true) - if err != nil { + if err := setkv(ctx, cscCol, in, c); err != nil { return nil, err } - return builtCertifyScorecard, nil + return c.buildScorecard(ctx, in, nil, true) } // Query CertifyScorecard @@ -152,32 +145,27 @@ func (c *demoClient) Scorecards(ctx context.Context, filter *model.CertifyScorec funcName := "Scorecards" if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*scorecardLink](id, c) + link, err := byIDkv[*scorecardLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - foundCertifyScorecard, err := c.buildScorecard(link, filter, true) + foundCertifyScorecard, err := c.buildScorecard(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.CertifyScorecard{foundCertifyScorecard}, nil } - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Source != nil { - exactSource, err := c.exactSource(filter.Source) + exactSource, err := c.exactSource(ctx, filter.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = exactSource.scorecardLinks + search = exactSource.ScorecardLinks foundOne = true } } @@ -185,19 +173,26 @@ func (c *demoClient) Scorecards(ctx context.Context, filter *model.CertifyScorec var out []*model.CertifyScorecard if foundOne { for _, id := range search { - link, err := byID[*scorecardLink](id, c) + link, err := byIDkv[*scorecardLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addSCIfMatch(out, filter, link) + out, err = c.addSCIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.scorecards { - var err error - out, err = c.addSCIfMatch(out, filter, link) + cscKeys, err := c.kv.Keys(ctx, cscCol) + if err != nil { + return nil, err + } + for _, csck := range cscKeys { + link, err := byKeykv[*scorecardLink](ctx, cscCol, csck, c) + if err != nil { + return nil, err + } + out, err = c.addSCIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -207,32 +202,32 @@ func (c *demoClient) Scorecards(ctx context.Context, filter *model.CertifyScorec return out, nil } -func (c *demoClient) addSCIfMatch(out []*model.CertifyScorecard, +func (c *demoClient) addSCIfMatch(ctx context.Context, out []*model.CertifyScorecard, filter *model.CertifyScorecardSpec, link *scorecardLink) ( []*model.CertifyScorecard, error) { - if filter != nil && filter.TimeScanned != nil && !filter.TimeScanned.Equal(link.timeScanned) { + if filter != nil && filter.TimeScanned != nil && !filter.TimeScanned.Equal(link.TimeScanned) { return out, nil } - if filter != nil && noMatchFloat(filter.AggregateScore, link.aggregateScore) { + if filter != nil && noMatchFloat(filter.AggregateScore, link.AggregateScore) { return out, nil } - if filter != nil && noMatchChecks(filter.Checks, link.checks) { + if filter != nil && noMatchChecks(filter.Checks, link.Checks) { return out, nil } - if filter != nil && noMatch(filter.ScorecardVersion, link.scorecardVersion) { + if filter != nil && noMatch(filter.ScorecardVersion, link.ScorecardVersion) { return out, nil } - if filter != nil && noMatch(filter.ScorecardCommit, link.scorecardCommit) { + if filter != nil && noMatch(filter.ScorecardCommit, link.ScorecardCommit) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - foundCertifyScorecard, err := c.buildScorecard(link, filter, false) + foundCertifyScorecard, err := c.buildScorecard(ctx, link, filter, false) if err != nil { return nil, err } @@ -242,16 +237,16 @@ func (c *demoClient) addSCIfMatch(out []*model.CertifyScorecard, return append(out, foundCertifyScorecard), nil } -func (c *demoClient) buildScorecard(link *scorecardLink, filter *model.CertifyScorecardSpec, ingestOrIDProvided bool) (*model.CertifyScorecard, error) { +func (c *demoClient) buildScorecard(ctx context.Context, link *scorecardLink, filter *model.CertifyScorecardSpec, ingestOrIDProvided bool) (*model.CertifyScorecard, error) { var s *model.Source var err error if filter != nil { - s, err = c.buildSourceResponse(link.sourceID, filter.Source) + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Source) if err != nil { return nil, err } } else { - s, err = c.buildSourceResponse(link.sourceID, nil) + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -265,16 +260,16 @@ func (c *demoClient) buildScorecard(link *scorecardLink, filter *model.CertifySc } newScorecard := model.CertifyScorecard{ - ID: nodeID(link.id), + ID: link.ThisID, Source: s, Scorecard: &model.Scorecard{ - TimeScanned: link.timeScanned, - AggregateScore: link.aggregateScore, - Checks: getCollectedScorecardChecks(link.checks), - ScorecardVersion: link.scorecardVersion, - ScorecardCommit: link.scorecardCommit, - Origin: link.origin, - Collector: link.collector, + TimeScanned: link.TimeScanned, + AggregateScore: link.AggregateScore, + Checks: getCollectedScorecardChecks(link.Checks), + ScorecardVersion: link.ScorecardVersion, + ScorecardCommit: link.ScorecardCommit, + Origin: link.Origin, + Collector: link.Collector, }, } return &newScorecard, nil diff --git a/pkg/assembler/backends/inmem/certifyScorecard_test.go b/pkg/assembler/backends/keyvalue/certifyScorecard_test.go similarity index 96% rename from pkg/assembler/backends/inmem/certifyScorecard_test.go rename to pkg/assembler/backends/keyvalue/certifyScorecard_test.go index 827f49f55a..febc7f755b 100644 --- a/pkg/assembler/backends/inmem/certifyScorecard_test.go +++ b/pkg/assembler/backends/keyvalue/certifyScorecard_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -383,25 +384,6 @@ func TestCertifyScorecard(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Src: s1, - SC: &model.ScorecardInputSpec{ - AggregateScore: 1.5, - TimeScanned: time.Unix(1e9, 0), - ScorecardVersion: "123", - ScorecardCommit: "abc", - }, - }, - }, - Query: &model.CertifyScorecardSpec{ - ID: ptrfrom.String("4294967296"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -409,7 +391,8 @@ func TestCertifyScorecard(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -591,7 +574,8 @@ func TestIngestScorecards(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -686,7 +670,8 @@ func TestCertifyScorecardNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/certifyVEXStatement.go b/pkg/assembler/backends/keyvalue/certifyVEXStatement.go new file mode 100644 index 0000000000..37bc7fea95 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/certifyVEXStatement.go @@ -0,0 +1,399 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal data: link between a package or an artifact with its corresponding +// vulnerability VEX statement +type vexLink struct { + ThisID string + PackageID string + ArtifactID string + VulnerabilityID string + KnownSince time.Time + Status model.VexStatus + Statement string + StatusNotes string + Justification model.VexJustification + Origin string + Collector string +} + +func (n *vexLink) ID() string { return n.ThisID } + +func (n *vexLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.ArtifactID, + n.VulnerabilityID, + timeKey(n.KnownSince), + string(n.Status), + n.Statement, + n.StatusNotes, + string(n.Justification), + n.Origin, + n.Collector, + }, ":") +} + +func (n *vexLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 2) + if n.PackageID != "" && allowedEdges[model.EdgeCertifyVexStatementPackage] { + out = append(out, n.PackageID) + } + if n.ArtifactID != "" && allowedEdges[model.EdgeCertifyVexStatementArtifact] { + out = append(out, n.ArtifactID) + } + if allowedEdges[model.EdgeCertifyVexStatementVulnerability] { + out = append(out, n.VulnerabilityID) + } + return out +} + +func (n *vexLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildCertifyVEXStatement(ctx, n, nil, true) +} + +// Ingest CertifyVex + +func (c *demoClient) IngestVEXStatements(ctx context.Context, subjects model.PackageOrArtifactInputs, vulnerabilities []*model.VulnerabilityInputSpec, vexStatements []*model.VexStatementInputSpec) ([]string, error) { + var modelVexStatementIDs []string + + for i := range vexStatements { + var certVex *model.CertifyVEXStatement + var err error + if len(subjects.Packages) > 0 { + subject := model.PackageOrArtifactInput{Package: subjects.Packages[i]} + certVex, err = c.IngestVEXStatement(ctx, subject, *vulnerabilities[i], *vexStatements[i]) + if err != nil { + return nil, gqlerror.Errorf("IngestVEXStatement failed with err: %v", err) + } + } else { + subject := model.PackageOrArtifactInput{Artifact: subjects.Artifacts[i]} + certVex, err = c.IngestVEXStatement(ctx, subject, *vulnerabilities[i], *vexStatements[i]) + if err != nil { + return nil, gqlerror.Errorf("IngestVEXStatement failed with err: %v", err) + } + } + modelVexStatementIDs = append(modelVexStatementIDs, certVex.ID) + } + return modelVexStatementIDs, nil +} + +func (c *demoClient) IngestVEXStatement(ctx context.Context, subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec) (*model.CertifyVEXStatement, error) { + return c.ingestVEXStatement(ctx, subject, vulnerability, vexStatement, true) +} + +func (c *demoClient) ingestVEXStatement(ctx context.Context, subject model.PackageOrArtifactInput, vulnerability model.VulnerabilityInputSpec, vexStatement model.VexStatementInputSpec, readOnly bool) (*model.CertifyVEXStatement, error) { + funcName := "IngestVEXStatement" + + in := &vexLink{ + KnownSince: vexStatement.KnownSince.UTC(), + Status: vexStatement.Status, + Statement: vexStatement.Statement, + StatusNotes: vexStatement.StatusNotes, + Justification: vexStatement.VexJustification, + Origin: vexStatement.Origin, + Collector: vexStatement.Collector, + } + + lock(&c.m, readOnly) + defer unlock(&c.m, readOnly) + + var foundPkgVersionNode *pkgVersion + var foundArtStrct *artStruct + if subject.Package != nil { + var err error + foundPkgVersionNode, err = c.getPackageVerFromInput(ctx, *subject.Package) + if err != nil { + return nil, gqlerror.Errorf("%v :: %s", funcName, err) + } + in.PackageID = foundPkgVersionNode.ThisID + } else { + var err error + foundArtStrct, err = c.artifactByInput(ctx, subject.Artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: %s", funcName, err) + } + in.ArtifactID = foundArtStrct.ThisID + } + + foundVulnNode, err := c.getVulnerabilityFromInput(ctx, vulnerability) + if err != nil { + return nil, gqlerror.Errorf("%v :: %s", funcName, err) + } + in.VulnerabilityID = foundVulnNode.ThisID + + out, err := byKeykv[*vexLink](ctx, cVEXCol, in.Key(), c) + if err == nil { + return c.buildCertifyVEXStatement(ctx, out, nil, true) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + + if readOnly { + c.m.RUnlock() + v, err := c.ingestVEXStatement(ctx, subject, vulnerability, vexStatement, false) + c.m.RLock() // relock so that defer unlock does not panic + return v, err + } + + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, cVEXCol, in); err != nil { + return nil, err + } + // set the backlinks + if foundPkgVersionNode != nil { + if err := foundPkgVersionNode.setVexLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + } else { + if err := foundArtStrct.setVexLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := foundVulnNode.setVexLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + if err := setkv(ctx, cVEXCol, in, c); err != nil { + return nil, err + } + + return c.buildCertifyVEXStatement(ctx, in, nil, true) +} + +// Query CertifyVex +func (c *demoClient) CertifyVEXStatement(ctx context.Context, filter *model.CertifyVEXStatementSpec) ([]*model.CertifyVEXStatement, error) { + c.m.RLock() + defer c.m.RUnlock() + funcName := "CertifyVEXStatement" + + if filter != nil && filter.ID != nil { + link, err := byIDkv[*vexLink](ctx, *filter.ID, c) + if err != nil { + // Not found + return nil, nil + } + // If found by id, ignore rest of fields in spec and return as a match + foundCertifyVex, err := c.buildCertifyVEXStatement(ctx, link, filter, true) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + return []*model.CertifyVEXStatement{foundCertifyVex}, nil + } + + var search []string + foundOne := false + + if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { + exactArtifact, err := c.artifactExact(ctx, filter.Subject.Artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + if exactArtifact != nil { + search = append(search, exactArtifact.VexLinks...) + foundOne = true + } + } + if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Package != nil { + pkgs, err := c.findPackageVersion(ctx, filter.Subject.Package) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + foundOne = len(pkgs) > 0 + for _, pkg := range pkgs { + search = append(search, pkg.VexLinks...) + } + } + if !foundOne && filter != nil && filter.Vulnerability != nil { + exactVuln, err := c.exactVulnerability(ctx, filter.Vulnerability) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + if exactVuln != nil { + search = append(search, exactVuln.VexLinks...) + foundOne = true + } + } + + var out []*model.CertifyVEXStatement + if foundOne { + for _, id := range search { + link, err := byIDkv[*vexLink](ctx, id, c) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + out, err = c.addVexIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } else { + keys, err := c.kv.Keys(ctx, cVEXCol) + if err != nil { + return nil, err + } + for _, key := range keys { + link, err := byKeykv[*vexLink](ctx, cVEXCol, key, c) + if err != nil { + return nil, err + } + out, err = c.addVexIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } + return out, nil +} + +func (c *demoClient) addVexIfMatch(ctx context.Context, out []*model.CertifyVEXStatement, + filter *model.CertifyVEXStatementSpec, link *vexLink) ( + []*model.CertifyVEXStatement, error) { + + if filter != nil && filter.KnownSince != nil && !filter.KnownSince.Equal(link.KnownSince) { + return out, nil + } + if filter != nil && filter.VexJustification != nil && *filter.VexJustification != link.Justification { + return out, nil + } + if filter != nil && filter.Status != nil && *filter.Status != link.Status { + return out, nil + } + if filter != nil && noMatch(filter.Statement, link.Statement) { + return out, nil + } + if filter != nil && noMatch(filter.StatusNotes, link.StatusNotes) { + return out, nil + } + if filter != nil && noMatch(filter.Collector, link.Collector) { + return out, nil + } + if filter != nil && noMatch(filter.Origin, link.Origin) { + return out, nil + } + + foundCertifyVex, err := c.buildCertifyVEXStatement(ctx, link, filter, false) + if err != nil { + return nil, err + } + if foundCertifyVex == nil { + return out, nil + } + return append(out, foundCertifyVex), nil + +} + +func (c *demoClient) buildCertifyVEXStatement(ctx context.Context, link *vexLink, filter *model.CertifyVEXStatementSpec, ingestOrIDProvided bool) (*model.CertifyVEXStatement, error) { + var p *model.Package + var a *model.Artifact + var vuln *model.Vulnerability + var err error + if filter != nil && filter.Subject != nil { + if filter.Subject.Package != nil && link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Subject.Package) + if err != nil { + return nil, err + } + } + if filter.Subject.Artifact != nil && link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, filter.Subject.Artifact) + if err != nil { + return nil, err + } + } + } else { + if link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) + if err != nil { + return nil, err + } + } + if link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, nil) + if err != nil { + return nil, err + } + } + } + + if filter != nil && filter.Vulnerability != nil { + if filter.Vulnerability != nil && link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, filter.Vulnerability) + if err != nil { + return nil, err + } + } + } else { + if link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, nil) + if err != nil { + return nil, err + } + } + } + + var subj model.PackageOrArtifact + if link.PackageID != "" { + if p == nil && ingestOrIDProvided { + return nil, gqlerror.Errorf("failed to retrieve package via packageID") + } else if p == nil && !ingestOrIDProvided { + return nil, nil + } + subj = p + } + if link.ArtifactID != "" { + if a == nil && ingestOrIDProvided { + return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") + } else if a == nil && !ingestOrIDProvided { + return nil, nil + } + subj = a + } + + if link.VulnerabilityID != "" { + if vuln == nil && ingestOrIDProvided { + return nil, gqlerror.Errorf("failed to retrieve vuln via vulnID") + } else if vuln == nil && !ingestOrIDProvided { + return nil, nil + } + } + + return &model.CertifyVEXStatement{ + ID: link.ThisID, + Subject: subj, + Vulnerability: vuln, + Status: link.Status, + VexJustification: link.Justification, + Statement: link.Statement, + StatusNotes: link.StatusNotes, + KnownSince: link.KnownSince, + Origin: link.Origin, + Collector: link.Collector, + }, nil +} diff --git a/pkg/assembler/backends/inmem/certifyVEXStatement_test.go b/pkg/assembler/backends/keyvalue/certifyVEXStatement_test.go similarity index 97% rename from pkg/assembler/backends/inmem/certifyVEXStatement_test.go rename to pkg/assembler/backends/keyvalue/certifyVEXStatement_test.go index 0fa26ec5ea..40eef9b336 100644 --- a/pkg/assembler/backends/inmem/certifyVEXStatement_test.go +++ b/pkg/assembler/backends/keyvalue/certifyVEXStatement_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -632,27 +633,6 @@ func TestVEX(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad id", - InPkg: []*model.PkgInputSpec{p1}, - InVuln: []*model.VulnerabilityInputSpec{o1}, - Calls: []call{ - { - Sub: model.PackageOrArtifactInput{ - Package: p1, - }, - Vuln: o1, - In: &model.VexStatementInputSpec{ - VexJustification: "test justification", - KnownSince: time.Unix(1e9, 0), - }, - }, - }, - Query: &model.CertifyVEXStatementSpec{ - ID: ptrfrom.String("-5"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -660,7 +640,8 @@ func TestVEX(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1024,7 +1005,8 @@ func TestVEXBulkIngest(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1137,7 +1119,8 @@ func TestVEXNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/certifyVuln.go b/pkg/assembler/backends/keyvalue/certifyVuln.go similarity index 51% rename from pkg/assembler/backends/inmem/certifyVuln.go rename to pkg/assembler/backends/keyvalue/certifyVuln.go index ca5ca848e1..87a496ae3a 100644 --- a/pkg/assembler/backends/inmem/certifyVuln.go +++ b/pkg/assembler/backends/keyvalue/certifyVuln.go @@ -13,50 +13,64 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" "reflect" - "strconv" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/internal/testing/ptrfrom" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link between packages and vulnerabilities (certifyVulnerability) -type certifyVulnerabilityList []*certifyVulnerabilityLink type certifyVulnerabilityLink struct { - id uint32 - packageID uint32 - vulnerabilityID uint32 - timeScanned time.Time - dbURI string - dbVersion string - scannerURI string - scannerVersion string - origin string - collector string + ThisID string + PackageID string + VulnerabilityID string + TimeScanned time.Time + DBURI string + DBVersion string + ScannerURI string + ScannerVersion string + Origin string + Collector string } -func (n *certifyVulnerabilityLink) ID() uint32 { return n.id } +func (n *certifyVulnerabilityLink) ID() string { return n.ThisID } +func (n *certifyVulnerabilityLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.VulnerabilityID, + timeKey(n.TimeScanned), + n.DBURI, + n.DBVersion, + n.ScannerURI, + n.ScannerVersion, + n.Origin, + n.Collector, + }, ":") +} -func (n *certifyVulnerabilityLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2) +func (n *certifyVulnerabilityLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 2) if allowedEdges[model.EdgeCertifyVulnPackage] { - out = append(out, n.packageID) + out = append(out, n.PackageID) } - if n.vulnerabilityID != 0 && allowedEdges[model.EdgeCertifyVulnVulnerability] { - out = append(out, n.vulnerabilityID) + if allowedEdges[model.EdgeCertifyVulnVulnerability] { + out = append(out, n.VulnerabilityID) } return out } -func (n *certifyVulnerabilityLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildCertifyVulnerability(n, nil, true) +func (n *certifyVulnerabilityLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildCertifyVulnerability(ctx, n, nil, true) } // Ingest CertifyVuln @@ -78,95 +92,63 @@ func (c *demoClient) IngestCertifyVuln(ctx context.Context, pkg model.PkgInputSp func (c *demoClient) ingestVulnerability(ctx context.Context, packageArg model.PkgInputSpec, vulnerability model.VulnerabilityInputSpec, certifyVuln model.ScanMetadataInput, readOnly bool) (*model.CertifyVuln, error) { funcName := "IngestVulnerability" + + in := &certifyVulnerabilityLink{ + TimeScanned: certifyVuln.TimeScanned.UTC(), + DBURI: certifyVuln.DbURI, + DBVersion: certifyVuln.DbVersion, + ScannerURI: certifyVuln.ScannerURI, + ScannerVersion: certifyVuln.ScannerVersion, + Origin: certifyVuln.Origin, + Collector: certifyVuln.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - packageID, err := getPackageIDFromInput(c, packageArg, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}) + foundPackage, err := c.getPackageVerFromInput(ctx, packageArg) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundPackage, err := byID[*pkgVersionNode](packageID, c) + in.PackageID = foundPackage.ThisID + + foundVulnNode, err := c.getVulnerabilityFromInput(ctx, vulnerability) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - packageVulns := foundPackage.certifyVulnLinks - - var vulnerabilityLinks []uint32 + in.VulnerabilityID = foundVulnNode.ThisID - vulnID, err := getVulnerabilityIDFromInput(c, vulnerability) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + out, err := byKeykv[*certifyVulnerabilityLink](ctx, cVulnCol, in.Key(), c) + if err == nil { + return c.buildCertifyVulnerability(ctx, out, nil, true) } - foundVulnNode, err := byID[*vulnIDNode](vulnID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + if !errors.Is(err, kv.NotFoundError) { + return nil, err } - vulnerabilityLinks = foundVulnNode.certifyVulnLinks - var searchIDs []uint32 - if len(packageVulns) < len(vulnerabilityLinks) { - searchIDs = packageVulns - } else { - searchIDs = vulnerabilityLinks + if readOnly { + c.m.RUnlock() + cv, err := c.ingestVulnerability(ctx, packageArg, vulnerability, certifyVuln, false) + c.m.RLock() // relock so that defer unlock does not panic + return cv, err } - // Don't insert duplicates - duplicate := false - collectedCertifyVulnLink := certifyVulnerabilityLink{} - for _, id := range searchIDs { - v, err := byID[*certifyVulnerabilityLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - vulnMatch := false - if vulnID != 0 && vulnID == v.vulnerabilityID { - vulnMatch = true - } - if vulnMatch && packageID == v.packageID && certifyVuln.TimeScanned.Equal(v.timeScanned) && certifyVuln.DbURI == v.dbURI && - certifyVuln.DbVersion == v.dbVersion && certifyVuln.ScannerURI == v.scannerURI && certifyVuln.ScannerVersion == v.scannerVersion && - certifyVuln.Origin == v.origin && certifyVuln.Collector == v.collector { - - collectedCertifyVulnLink = *v - duplicate = true - break - } + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, cVulnCol, in); err != nil { + return nil, err } - - if !duplicate { - if readOnly { - c.m.RUnlock() - cv, err := c.ingestVulnerability(ctx, packageArg, vulnerability, certifyVuln, false) - c.m.RLock() // relock so that defer unlock does not panic - return cv, err - } - // store the link - collectedCertifyVulnLink = certifyVulnerabilityLink{ - id: c.getNextID(), - packageID: packageID, - vulnerabilityID: vulnID, - timeScanned: certifyVuln.TimeScanned, - dbURI: certifyVuln.DbURI, - dbVersion: certifyVuln.DbVersion, - scannerURI: certifyVuln.ScannerURI, - scannerVersion: certifyVuln.ScannerVersion, - origin: certifyVuln.Origin, - collector: certifyVuln.Collector, - } - c.index[collectedCertifyVulnLink.id] = &collectedCertifyVulnLink - c.certifyVulnerabilities = append(c.certifyVulnerabilities, &collectedCertifyVulnLink) - // set the backlinks - foundPackage.setVulnerabilityLinks(collectedCertifyVulnLink.id) - if vulnID != 0 { - foundVulnNode.setVulnerabilityLinks(collectedCertifyVulnLink.id) - } + // set the backlinks + if err := foundPackage.setVulnerabilityLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - - // build return GraphQL type - builtCertifyVuln, err := c.buildCertifyVulnerability(&collectedCertifyVulnLink, nil, true) - if err != nil { + if err := foundVulnNode.setVulnerabilityLinks(ctx, in.ThisID, c); err != nil { return nil, err } - return builtCertifyVuln, nil + if err := setkv(ctx, cVulnCol, in, c); err != nil { + return nil, err + } + + return c.buildCertifyVulnerability(ctx, in, nil, true) } // Query CertifyVuln @@ -176,40 +158,36 @@ func (c *demoClient) CertifyVuln(ctx context.Context, filter *model.CertifyVulnS funcName := "CertifyVuln" if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*certifyVulnerabilityLink](id, c) + link, err := byIDkv[*certifyVulnerabilityLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - foundCertifyVuln, err := c.buildCertifyVulnerability(link, filter, true) + foundCertifyVuln, err := c.buildCertifyVulnerability(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.CertifyVuln{foundCertifyVuln}, nil } - var search []uint32 + var search []string foundOne := false + if filter != nil && filter.Package != nil { - pkgs, err := c.findPackageVersion(filter.Package) + pkgs, err := c.findPackageVersion(ctx, filter.Package) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } foundOne = len(pkgs) > 0 for _, pkg := range pkgs { - search = append(search, pkg.certifyVulnLinks...) + search = append(search, pkg.CertifyVulnLinks...) } } + if !foundOne && filter != nil && filter.Vulnerability != nil && filter.Vulnerability.NoVuln != nil && *filter.Vulnerability.NoVuln { - - exactVuln, err := c.exactVulnerability(&model.VulnerabilitySpec{ + exactVuln, err := c.exactVulnerability(ctx, &model.VulnerabilitySpec{ Type: ptrfrom.String(noVulnType), VulnerabilityID: ptrfrom.String(""), }) @@ -217,23 +195,21 @@ func (c *demoClient) CertifyVuln(ctx context.Context, filter *model.CertifyVulnS return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactVuln != nil { - search = append(search, exactVuln.certifyVulnLinks...) + search = append(search, exactVuln.CertifyVulnLinks...) foundOne = true } } else if !foundOne && filter != nil && filter.Vulnerability != nil { - if filter.Vulnerability.NoVuln != nil && !*filter.Vulnerability.NoVuln { if filter.Vulnerability.Type != nil && *filter.Vulnerability.Type == noVulnType { return []*model.CertifyVuln{}, gqlerror.Errorf("novuln boolean set to false, cannot specify vulnerability type to be novuln") } } - - exactVuln, err := c.exactVulnerability(filter.Vulnerability) + exactVuln, err := c.exactVulnerability(ctx, filter.Vulnerability) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactVuln != nil { - search = append(search, exactVuln.certifyVulnLinks...) + search = append(search, exactVuln.CertifyVulnLinks...) foundOne = true } } @@ -241,19 +217,26 @@ func (c *demoClient) CertifyVuln(ctx context.Context, filter *model.CertifyVulnS var out []*model.CertifyVuln if foundOne { for _, id := range search { - link, err := byID[*certifyVulnerabilityLink](id, c) + link, err := byIDkv[*certifyVulnerabilityLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addCVIfMatch(out, filter, link) + out, err = c.addCVIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.certifyVulnerabilities { - var err error - out, err = c.addCVIfMatch(out, filter, link) + keys, err := c.kv.Keys(ctx, cVulnCol) + if err != nil { + return nil, err + } + for _, key := range keys { + link, err := byKeykv[*certifyVulnerabilityLink](ctx, cVulnCol, key, c) + if err != nil { + return nil, err + } + out, err = c.addCVIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -263,32 +246,32 @@ func (c *demoClient) CertifyVuln(ctx context.Context, filter *model.CertifyVulnS return out, nil } -func (c *demoClient) addCVIfMatch(out []*model.CertifyVuln, +func (c *demoClient) addCVIfMatch(ctx context.Context, out []*model.CertifyVuln, filter *model.CertifyVulnSpec, link *certifyVulnerabilityLink) ([]*model.CertifyVuln, error) { - if filter != nil && filter.TimeScanned != nil && !filter.TimeScanned.Equal(link.timeScanned) { + if filter != nil && filter.TimeScanned != nil && !filter.TimeScanned.Equal(link.TimeScanned) { return out, nil } - if filter != nil && noMatch(filter.DbURI, link.dbURI) { + if filter != nil && noMatch(filter.DbURI, link.DBURI) { return out, nil } - if filter != nil && noMatch(filter.DbVersion, link.dbVersion) { + if filter != nil && noMatch(filter.DbVersion, link.DBVersion) { return out, nil } - if filter != nil && noMatch(filter.ScannerURI, link.scannerURI) { + if filter != nil && noMatch(filter.ScannerURI, link.ScannerURI) { return out, nil } - if filter != nil && noMatch(filter.ScannerVersion, link.scannerVersion) { + if filter != nil && noMatch(filter.ScannerVersion, link.ScannerVersion) { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - foundCertifyVuln, err := c.buildCertifyVulnerability(link, filter, false) + foundCertifyVuln, err := c.buildCertifyVulnerability(ctx, link, filter, false) if err != nil { return nil, err } @@ -298,25 +281,25 @@ func (c *demoClient) addCVIfMatch(out []*model.CertifyVuln, return append(out, foundCertifyVuln), nil } -func (c *demoClient) buildCertifyVulnerability(link *certifyVulnerabilityLink, filter *model.CertifyVulnSpec, ingestOrIDProvided bool) (*model.CertifyVuln, error) { +func (c *demoClient) buildCertifyVulnerability(ctx context.Context, link *certifyVulnerabilityLink, filter *model.CertifyVulnSpec, ingestOrIDProvided bool) (*model.CertifyVuln, error) { var p *model.Package var vuln *model.Vulnerability var err error if filter != nil { - p, err = c.buildPackageResponse(link.packageID, filter.Package) + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Package) if err != nil { return nil, err } } else { - p, err = c.buildPackageResponse(link.packageID, nil) + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } } if filter != nil && filter.Vulnerability != nil { - if filter.Vulnerability != nil && link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, filter.Vulnerability) + if filter.Vulnerability != nil && link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, filter.Vulnerability) if err != nil { return nil, err } @@ -329,8 +312,8 @@ func (c *demoClient) buildCertifyVulnerability(link *certifyVulnerabilityLink, f } } } else { - if link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, nil) + if link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, nil) if err != nil { return nil, err } @@ -344,7 +327,7 @@ func (c *demoClient) buildCertifyVulnerability(link *certifyVulnerabilityLink, f return nil, nil } - if link.vulnerabilityID != 0 { + if link.VulnerabilityID != "" { if vuln == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve vuln via vulnID") } else if vuln == nil && !ingestOrIDProvided { @@ -352,21 +335,18 @@ func (c *demoClient) buildCertifyVulnerability(link *certifyVulnerabilityLink, f } } - metadata := &model.ScanMetadata{ - TimeScanned: link.timeScanned, - DbURI: link.dbURI, - DbVersion: link.dbVersion, - ScannerURI: link.scannerURI, - ScannerVersion: link.scannerVersion, - Origin: link.origin, - Collector: link.collector, - } - - certifyVuln := model.CertifyVuln{ - ID: nodeID(link.id), + return &model.CertifyVuln{ + ID: link.ThisID, Package: p, Vulnerability: vuln, - Metadata: metadata, - } - return &certifyVuln, nil + Metadata: &model.ScanMetadata{ + TimeScanned: link.TimeScanned, + DbURI: link.DBURI, + DbVersion: link.DBVersion, + ScannerURI: link.ScannerURI, + ScannerVersion: link.ScannerVersion, + Origin: link.Origin, + Collector: link.Collector, + }, + }, nil } diff --git a/pkg/assembler/backends/inmem/certifyVuln_test.go b/pkg/assembler/backends/keyvalue/certifyVuln_test.go similarity index 98% rename from pkg/assembler/backends/inmem/certifyVuln_test.go rename to pkg/assembler/backends/keyvalue/certifyVuln_test.go index 5a87544e04..c7db555055 100644 --- a/pkg/assembler/backends/inmem/certifyVuln_test.go +++ b/pkg/assembler/backends/keyvalue/certifyVuln_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -25,6 +25,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -600,7 +601,8 @@ func TestIngestCertifyVulnerability(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -699,20 +701,18 @@ func TestIngestCertifyVulns(t *testing.T) { }, ExpVuln: []*model.CertifyVuln{ { - ID: "1", - Package: p2out, + Package: p1out, Vulnerability: &model.Vulnerability{ Type: "cve", - VulnerabilityIDs: []*model.VulnerabilityID{c1out}, + VulnerabilityIDs: []*model.VulnerabilityID{c2out}, }, Metadata: vmd1, }, { - ID: "10", - Package: p1out, + Package: p2out, Vulnerability: &model.Vulnerability{ Type: "cve", - VulnerabilityIDs: []*model.VulnerabilityID{c2out}, + VulnerabilityIDs: []*model.VulnerabilityID{c1out}, }, Metadata: vmd1, }, @@ -1065,7 +1065,8 @@ func TestIngestCertifyVulns(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1184,7 +1185,8 @@ func TestCertifyVulnNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/hasMetadata.go b/pkg/assembler/backends/keyvalue/hasMetadata.go similarity index 50% rename from pkg/assembler/backends/inmem/hasMetadata.go rename to pkg/assembler/backends/keyvalue/hasMetadata.go index 3c90646cd6..2809448de7 100644 --- a/pkg/assembler/backends/inmem/hasMetadata.go +++ b/pkg/assembler/backends/keyvalue/hasMetadata.go @@ -13,51 +13,64 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) -// Internal data: link that a package/source/artifact is good -type hasMetadataList []*hasMetadataLink type hasMetadataLink struct { - id uint32 - packageID uint32 - artifactID uint32 - sourceID uint32 - timestamp time.Time - key string - value string - justification string - origin string - collector string + ThisID string + PackageID string + ArtifactID string + SourceID string + Timestamp time.Time + MDKey string + Value string + Justification string + Origin string + Collector string } -func (n *hasMetadataLink) ID() uint32 { return n.id } +func (n *hasMetadataLink) ID() string { return n.ThisID } +func (n *hasMetadataLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.ArtifactID, + n.SourceID, + timeKey(n.Timestamp), + n.MDKey, + n.Value, + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *hasMetadataLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 1) - if n.packageID != 0 && allowedEdges[model.EdgeHasMetadataPackage] { - out = append(out, n.packageID) +func (n *hasMetadataLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 1) + if n.PackageID != "" && allowedEdges[model.EdgeHasMetadataPackage] { + out = append(out, n.PackageID) } - if n.artifactID != 0 && allowedEdges[model.EdgeHasMetadataArtifact] { - out = append(out, n.artifactID) + if n.ArtifactID != "" && allowedEdges[model.EdgeHasMetadataArtifact] { + out = append(out, n.ArtifactID) } - if n.sourceID != 0 && allowedEdges[model.EdgeHasMetadataSource] { - out = append(out, n.sourceID) + if n.SourceID != "" && allowedEdges[model.EdgeHasMetadataSource] { + out = append(out, n.SourceID) } return out } -func (n *hasMetadataLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildHasMetadata(n, nil, true) +func (n *hasMetadataLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildHasMetadata(ctx, n, nil, true) } // Ingest HasMetadata @@ -99,120 +112,87 @@ func (c *demoClient) IngestHasMetadata(ctx context.Context, subject model.Packag func (c *demoClient) ingestHasMetadata(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, hasMetadata model.HasMetadataInputSpec, readOnly bool) (*model.HasMetadata, error) { funcName := "IngestHasMetadata" + in := &hasMetadataLink{ + MDKey: hasMetadata.Key, + Value: hasMetadata.Value, + Timestamp: hasMetadata.Timestamp.UTC(), + Justification: hasMetadata.Justification, + Origin: hasMetadata.Origin, + Collector: hasMetadata.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var packageID uint32 var foundPkgNameorVersionNode pkgNameOrVersion - var artifactID uint32 var foundArtStrct *artStruct - var sourceID uint32 var srcName *srcNameNode - searchIDs := []uint32{} if subject.Package != nil { var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, *pkgMatchType) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundPkgNameorVersionNode, err = byID[pkgNameOrVersion](packageID, c) + foundPkgNameorVersionNode, err = c.getPackageNameOrVerFromInput(ctx, *subject.Package, *pkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - searchIDs = append(searchIDs, foundPkgNameorVersionNode.getHasMetadataLinks()...) + in.PackageID = foundPkgNameorVersionNode.ID() } else if subject.Artifact != nil { var err error - artifactID, err = getArtifactIDFromInput(c, *subject.Artifact) + foundArtStrct, err = c.artifactByInput(ctx, subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundArtStrct, err = byID[*artStruct](artifactID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = append(searchIDs, foundArtStrct.hasMetadataLinks...) + in.ArtifactID = foundArtStrct.ThisID } else { var err error - sourceID, err = getSourceIDFromInput(c, *subject.Source) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - srcName, err = byID[*srcNameNode](sourceID, c) + srcName, err = c.getSourceNameFromInput(ctx, *subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - searchIDs = append(searchIDs, srcName.hasMetadataLinks...) + in.SourceID = srcName.ThisID } - // Don't insert duplicates - duplicate := false - collectedLink := hasMetadataLink{} - for _, id := range searchIDs { - v, err := byID[*hasMetadataLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectMatch := false - if packageID != 0 && packageID == v.packageID { - subjectMatch = true - } - if artifactID != 0 && artifactID == v.artifactID { - subjectMatch = true - } - if sourceID != 0 && sourceID == v.sourceID { - subjectMatch = true - } - if subjectMatch && hasMetadata.Justification == v.justification && - hasMetadata.Key == v.key && hasMetadata.Value == v.value && - hasMetadata.Timestamp.Equal(v.timestamp) && - hasMetadata.Origin == v.origin && hasMetadata.Collector == v.collector { - - collectedLink = *v - duplicate = true - break - } + out, err := byKeykv[*hasMetadataLink](ctx, hasMDCol, in.Key(), c) + if err == nil { + return c.buildHasMetadata(ctx, out, nil, true) } - if !duplicate { - if readOnly { - c.m.RUnlock() - b, err := c.ingestHasMetadata(ctx, subject, pkgMatchType, hasMetadata, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - // store the link - collectedLink = hasMetadataLink{ - id: c.getNextID(), - packageID: packageID, - artifactID: artifactID, - sourceID: sourceID, - key: hasMetadata.Key, - value: hasMetadata.Value, - timestamp: hasMetadata.Timestamp, - justification: hasMetadata.Justification, - origin: hasMetadata.Origin, - collector: hasMetadata.Collector, - } - c.index[collectedLink.id] = &collectedLink - c.hasMetadatas = append(c.hasMetadatas, &collectedLink) - // set the backlinks - if packageID != 0 { - foundPkgNameorVersionNode.setHasMetadataLinks(collectedLink.id) + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + + if readOnly { + c.m.RUnlock() + b, err := c.ingestHasMetadata(ctx, subject, pkgMatchType, hasMetadata, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err + } + + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, hasMDCol, in); err != nil { + return nil, err + } + + // set the backlinks + if foundPkgNameorVersionNode != nil { + if err := foundPkgNameorVersionNode.setHasMetadataLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if artifactID != 0 { - foundArtStrct.setHasMetadataLinks(collectedLink.id) + } + if foundArtStrct != nil { + if err := foundArtStrct.setHasMetadataLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if sourceID != 0 { - srcName.setHasMetadataLinks(collectedLink.id) + } + if srcName != nil { + if err := srcName.setHasMetadataLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - } - // build return GraphQL type - builtHasMetadata, err := c.buildHasMetadata(&collectedLink, nil, true) - if err != nil { + if err := setkv(ctx, hasMDCol, in, c); err != nil { return nil, err } - return builtHasMetadata, nil + + // build return GraphQL type + return c.buildHasMetadata(ctx, in, nil, true) } // Query HasMetadata @@ -223,17 +203,12 @@ func (c *demoClient) HasMetadata(ctx context.Context, filter *model.HasMetadataS defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*hasMetadataLink](id, c) + link, err := byIDkv[*hasMetadataLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - found, err := c.buildHasMetadata(link, filter, true) + found, err := c.buildHasMetadata(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -242,25 +217,25 @@ func (c *demoClient) HasMetadata(ctx context.Context, filter *model.HasMetadataS // Cant really search for an exact Pkg, as these can be linked to either // names or versions, and version could be empty. - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Subject.Artifact) + exactArtifact, err := c.artifactExact(ctx, filter.Subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.hasMetadataLinks...) + search = append(search, exactArtifact.HasMetadataLinks...) foundOne = true } } if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) + exactSource, err := c.exactSource(ctx, filter.Subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.hasMetadataLinks...) + search = append(search, exactSource.HasMetadataLinks...) foundOne = true } } @@ -268,19 +243,26 @@ func (c *demoClient) HasMetadata(ctx context.Context, filter *model.HasMetadataS var out []*model.HasMetadata if foundOne { for _, id := range search { - link, err := byID[*hasMetadataLink](id, c) + link, err := byIDkv[*hasMetadataLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addHMIfMatch(out, filter, link) + out, err = c.addHMIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.hasMetadatas { - var err error - out, err = c.addHMIfMatch(out, filter, link) + hmk, err := c.kv.Keys(ctx, hasMDCol) + if err != nil { + return nil, err + } + for _, hk := range hmk { + link, err := byKeykv[*hasMetadataLink](ctx, hasMDCol, hk, c) + if err != nil { + return nil, err + } + out, err = c.addHMIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -289,30 +271,30 @@ func (c *demoClient) HasMetadata(ctx context.Context, filter *model.HasMetadataS return out, nil } -func (c *demoClient) addHMIfMatch(out []*model.HasMetadata, filter *model.HasMetadataSpec, link *hasMetadataLink) ( +func (c *demoClient) addHMIfMatch(ctx context.Context, out []*model.HasMetadata, filter *model.HasMetadataSpec, link *hasMetadataLink) ( []*model.HasMetadata, error) { - if filter != nil && noMatch(filter.Justification, link.justification) { + if filter != nil && noMatch(filter.Justification, link.Justification) { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - if filter != nil && noMatch(filter.Key, link.key) { + if filter != nil && noMatch(filter.Key, link.MDKey) { return out, nil } - if filter != nil && noMatch(filter.Value, link.value) { + if filter != nil && noMatch(filter.Value, link.Value) { return out, nil } // no match if filter time since is after the timestamp - if filter != nil && filter.Since != nil && filter.Since.After(link.timestamp) { + if filter != nil && filter.Since != nil && filter.Since.After(link.Timestamp) { return out, nil } - found, err := c.buildHasMetadata(link, filter, false) + found, err := c.buildHasMetadata(ctx, link, filter, false) if err != nil { return nil, err } @@ -322,45 +304,45 @@ func (c *demoClient) addHMIfMatch(out []*model.HasMetadata, filter *model.HasMet return append(out, found), nil } -func (c *demoClient) buildHasMetadata(link *hasMetadataLink, filter *model.HasMetadataSpec, ingestOrIDProvided bool) (*model.HasMetadata, error) { +func (c *demoClient) buildHasMetadata(ctx context.Context, link *hasMetadataLink, filter *model.HasMetadataSpec, ingestOrIDProvided bool) (*model.HasMetadata, error) { var p *model.Package var a *model.Artifact var s *model.Source var err error if filter != nil && filter.Subject != nil { - if filter.Subject.Package != nil && link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, filter.Subject.Package) + if filter.Subject.Package != nil && link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Subject.Package) if err != nil { return nil, err } } - if filter.Subject.Artifact != nil && link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, filter.Subject.Artifact) + if filter.Subject.Artifact != nil && link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, filter.Subject.Artifact) if err != nil { return nil, err } } - if filter.Subject.Source != nil && link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, filter.Subject.Source) + if filter.Subject.Source != nil && link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Subject.Source) if err != nil { return nil, err } } } else { - if link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, nil) + if link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } } - if link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, nil) + if link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, nil) if err != nil { return nil, err } } - if link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, nil) + if link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -368,7 +350,7 @@ func (c *demoClient) buildHasMetadata(link *hasMetadataLink, filter *model.HasMe } var subj model.PackageSourceOrArtifact - if link.packageID != 0 { + if link.PackageID != "" { if p == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve package via packageID") } else if p == nil && !ingestOrIDProvided { @@ -376,7 +358,7 @@ func (c *demoClient) buildHasMetadata(link *hasMetadataLink, filter *model.HasMe } subj = p } - if link.artifactID != 0 { + if link.ArtifactID != "" { if a == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") } else if a == nil && !ingestOrIDProvided { @@ -384,7 +366,7 @@ func (c *demoClient) buildHasMetadata(link *hasMetadataLink, filter *model.HasMe } subj = a } - if link.sourceID != 0 { + if link.SourceID != "" { if s == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve source via sourceID") } else if s == nil && !ingestOrIDProvided { @@ -394,14 +376,14 @@ func (c *demoClient) buildHasMetadata(link *hasMetadataLink, filter *model.HasMe } hasMetadata := model.HasMetadata{ - ID: nodeID(link.id), + ID: link.ThisID, Subject: subj, - Timestamp: link.timestamp, - Key: link.key, - Value: link.value, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, + Timestamp: link.Timestamp, + Key: link.MDKey, + Value: link.Value, + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, } return &hasMetadata, nil } diff --git a/pkg/assembler/backends/inmem/hasMetadata_test.go b/pkg/assembler/backends/keyvalue/hasMetadata_test.go similarity index 97% rename from pkg/assembler/backends/inmem/hasMetadata_test.go rename to pkg/assembler/backends/keyvalue/hasMetadata_test.go index b3cc561e7d..6a84f8ba17 100644 --- a/pkg/assembler/backends/inmem/hasMetadata_test.go +++ b/pkg/assembler/backends/keyvalue/hasMetadata_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -545,11 +546,11 @@ func TestHasMetadata(t *testing.T) { }, ExpHM: []*model.HasMetadata{ { - Subject: p2out, + Subject: p1outName, Justification: "test justification", }, { - Subject: p1outName, + Subject: p2out, Justification: "test justification", }, }, @@ -599,24 +600,6 @@ func TestHasMetadata(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query good ID", - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Sub: model.PackageSourceOrArtifactInput{ - Source: s1, - }, - HM: &model.HasMetadataInputSpec{ - Justification: "test justification", - }, - }, - }, - Query: &model.HasMetadataSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -624,7 +607,8 @@ func TestHasMetadata(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -912,7 +896,8 @@ func TestIngestBulkHasMetadata(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1041,7 +1026,8 @@ func TestHasMetadataNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/hasSBOM.go b/pkg/assembler/backends/keyvalue/hasSBOM.go new file mode 100644 index 0000000000..d2a93299ef --- /dev/null +++ b/pkg/assembler/backends/keyvalue/hasSBOM.go @@ -0,0 +1,443 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +type hasSBOMStruct struct { + ThisID string + Pkg string + Artifact string + URI string + Algorithm string + Digest string + DownloadLocation string + Origin string + Collector string + KnownSince time.Time + IncludedSoftware []string + IncludedDependencies []string + IncludedOccurrences []string +} + +func (n *hasSBOMStruct) ID() string { return n.ThisID } +func (n *hasSBOMStruct) Key() string { + return strings.Join([]string{ + n.Pkg, + n.Artifact, + n.URI, + n.Algorithm, + n.Digest, + n.DownloadLocation, + n.Origin, + n.Collector, + timeKey(n.KnownSince), + fmt.Sprint(n.IncludedSoftware), + fmt.Sprint(n.IncludedDependencies), + fmt.Sprint(n.IncludedOccurrences), + }, ":") +} + +func (n *hasSBOMStruct) Neighbors(allowedEdges edgeMap) []string { + var out []string + if n.Pkg != "" && allowedEdges[model.EdgeHasSbomPackage] { + out = append(out, n.Pkg) + } + if n.Artifact != "" && allowedEdges[model.EdgeHasSbomArtifact] { + out = append(out, n.Artifact) + } + if allowedEdges[model.EdgeHasSbomIncludedSoftware] { + out = append(out, n.IncludedSoftware...) + } + if allowedEdges[model.EdgeHasSbomIncludedDependencies] { + out = append(out, n.IncludedDependencies...) + } + if allowedEdges[model.EdgeHasSbomIncludedOccurrences] { + out = append(out, n.IncludedOccurrences...) + } + return sortAndRemoveDups(out) +} + +func (n *hasSBOMStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convHasSBOM(ctx, n) +} + +// Ingest HasSBOM + +func (c *demoClient) IngestHasSBOMs(ctx context.Context, subjects model.PackageOrArtifactInputs, hasSBOMs []*model.HasSBOMInputSpec, includes []*model.HasSBOMIncludesInputSpec) ([]*model.HasSbom, error) { + var modelHasSboms []*model.HasSbom + + for i := range hasSBOMs { + var hasSBOM *model.HasSbom + var err error + if len(subjects.Packages) > 0 { + subject := model.PackageOrArtifactInput{Package: subjects.Packages[i]} + hasSBOM, err = c.IngestHasSbom(ctx, subject, *hasSBOMs[i], *includes[i]) + if err != nil { + return nil, gqlerror.Errorf("IngestHasSbom failed with err: %v", err) + } + } else { + subject := model.PackageOrArtifactInput{Artifact: subjects.Artifacts[i]} + hasSBOM, err = c.IngestHasSbom(ctx, subject, *hasSBOMs[i], *includes[i]) + if err != nil { + return nil, gqlerror.Errorf("IngestHasSbom failed with err: %v", err) + } + } + modelHasSboms = append(modelHasSboms, hasSBOM) + } + return modelHasSboms, nil +} + +func (c *demoClient) IngestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, input model.HasSBOMInputSpec, includes model.HasSBOMIncludesInputSpec) (*model.HasSbom, error) { + funcName := "IngestHasSbom" + + c.m.RLock() + for _, id := range includes.Software { + if err := c.validateSoftwareId(ctx, funcName, id); err != nil { + c.m.RUnlock() + return nil, err + } + } + for _, id := range includes.Dependencies { + if _, err := byIDkv[*isDependencyLink](ctx, id, c); err != nil { + c.m.RUnlock() + return nil, gqlerror.Errorf("%v :: dependency id %v is not an ingested isDependency", funcName, id) + } + } + for _, id := range includes.Occurrences { + if _, err := byIDkv[*isOccurrenceStruct](ctx, id, c); err != nil { + c.m.RUnlock() + return nil, gqlerror.Errorf("%v :: occurrence id %v is not an ingested isOccurrence", funcName, id) + } + } + c.m.RUnlock() + + softwareIDs := sortAndRemoveDups(includes.Software) + dependencyIDs := sortAndRemoveDups(includes.Dependencies) + occurrenceIDs := sortAndRemoveDups(includes.Occurrences) + return c.ingestHasSbom(ctx, subject, input, softwareIDs, dependencyIDs, occurrenceIDs, true) +} + +func (c *demoClient) validateSoftwareId(ctx context.Context, funcName string, id string) error { + if _, err := byIDkv[*pkgVersion](ctx, id, c); err != nil { + if _, err := byIDkv[*artStruct](ctx, id, c); err != nil { + return gqlerror.Errorf("%v :: software id %v is neither an ingested Package nor an ingested Artifact", funcName, id) + } + } + return nil +} + +func (c *demoClient) ingestHasSbom(ctx context.Context, subject model.PackageOrArtifactInput, input model.HasSBOMInputSpec, includedSoftware, includedDependencies, includedOccurrences []string, readOnly bool) (*model.HasSbom, error) { + funcName := "IngestHasSbom" + algorithm := strings.ToLower(input.Algorithm) + digest := strings.ToLower(input.Digest) + + in := &hasSBOMStruct{ + URI: input.URI, + Algorithm: algorithm, + Digest: digest, + DownloadLocation: input.DownloadLocation, + Origin: input.Origin, + Collector: input.Collector, + KnownSince: input.KnownSince.UTC(), + IncludedSoftware: includedSoftware, + IncludedDependencies: includedDependencies, + IncludedOccurrences: includedOccurrences, + } + + lock(&c.m, readOnly) + defer unlock(&c.m, readOnly) + + var pkg *pkgVersion + var art *artStruct + + if subject.Package != nil { + var err error + pkg, err = c.getPackageVerFromInput(ctx, *subject.Package) + if err != nil { + return nil, gqlerror.Errorf("%v :: %s", funcName, err) + } + in.Pkg = pkg.ThisID + } else { + var err error + art, err = c.artifactByInput(ctx, subject.Artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: %s", funcName, err) + } + in.Artifact = art.ThisID + } + + out, err := byKeykv[*hasSBOMStruct](ctx, hasSBOMCol, in.Key(), c) + if err == nil { + return c.convHasSBOM(ctx, out) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + + if readOnly { + c.m.RUnlock() + b, err := c.ingestHasSbom(ctx, subject, input, includedSoftware, includedDependencies, includedOccurrences, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err + } + + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, hasSBOMCol, in); err != nil { + return nil, err + } + + if pkg != nil { + if err := pkg.setHasSBOM(ctx, in.ThisID, c); err != nil { + return nil, err + } + } else { + if err := art.setHasSBOMs(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + + if err := setkv(ctx, hasSBOMCol, in, c); err != nil { + return nil, err + } + + return c.convHasSBOM(ctx, in) +} + +func (c *demoClient) convHasSBOM(ctx context.Context, in *hasSBOMStruct) (*model.HasSbom, error) { + out := &model.HasSbom{ + ID: in.ThisID, + URI: in.URI, + Algorithm: in.Algorithm, + Digest: in.Digest, + DownloadLocation: in.DownloadLocation, + Origin: in.Origin, + Collector: in.Collector, + KnownSince: in.KnownSince.UTC(), + } + if in.Pkg != "" { + p, err := c.buildPackageResponse(ctx, in.Pkg, nil) + if err != nil { + return nil, err + } + out.Subject = p + } else { + art, err := c.artifactModelByID(ctx, in.Artifact) + if err != nil { + return nil, err + } + out.Subject = art + } + if len(in.IncludedSoftware) > 0 { + out.IncludedSoftware = make([]model.PackageOrArtifact, 0, len(in.IncludedSoftware)) + for _, id := range in.IncludedSoftware { + p, err := c.buildPackageResponse(ctx, id, nil) + if err != nil { + art, err := c.artifactModelByID(ctx, id) + if err != nil { + return nil, fmt.Errorf("expected Package or Artifact: %w", err) + } + out.IncludedSoftware = append(out.IncludedSoftware, art) + } else { + out.IncludedSoftware = append(out.IncludedSoftware, p) + } + } + } + if len(in.IncludedDependencies) > 0 { + out.IncludedDependencies = make([]*model.IsDependency, 0, len(in.IncludedDependencies)) + for _, id := range in.IncludedDependencies { + link, err := byIDkv[*isDependencyLink](ctx, id, c) + if err != nil { + return nil, fmt.Errorf("expected IsDependency: %w", err) + } + isDep, err := c.buildIsDependency(ctx, link, nil, true) + if err != nil { + return nil, err + } + out.IncludedDependencies = append(out.IncludedDependencies, isDep) + } + } + if len(in.IncludedOccurrences) > 0 { + out.IncludedOccurrences = make([]*model.IsOccurrence, 0, len(in.IncludedOccurrences)) + for _, id := range in.IncludedOccurrences { + link, err := byIDkv[*isOccurrenceStruct](ctx, id, c) + if err != nil { + return nil, fmt.Errorf("expected IsDependency: %w", err) + } + isOcc, err := c.convOccurrence(ctx, link) + if err != nil { + return nil, err + } + out.IncludedOccurrences = append(out.IncludedOccurrences, isOcc) + } + } + return out, nil +} + +// Query HasSBOM + +func (c *demoClient) HasSBOM(ctx context.Context, filter *model.HasSBOMSpec) ([]*model.HasSbom, error) { + funcName := "HasSBOM" + c.m.RLock() + defer c.m.RUnlock() + + if filter != nil && filter.ID != nil { + link, err := byIDkv[*hasSBOMStruct](ctx, *filter.ID, c) + if err != nil { + // Not found + return nil, nil + } + // If found by id, ignore rest of fields in spec and return as a match + sb, err := c.convHasSBOM(ctx, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + return []*model.HasSbom{sb}, nil + } + + var search []string + foundOne := false + if filter != nil && filter.Subject != nil && filter.Subject.Package != nil { + pkgs, err := c.findPackageVersion(ctx, filter.Subject.Package) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + foundOne = len(pkgs) > 0 + for _, pkg := range pkgs { + search = append(search, pkg.HasSBOMs...) + } + } + if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { + exactArt, err := c.artifactExact(ctx, filter.Subject.Artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + if exactArt != nil { + search = exactArt.HasSBOMs + foundOne = true + } + } + + var out []*model.HasSbom + if foundOne { + for _, id := range search { + link, err := byIDkv[*hasSBOMStruct](ctx, id, c) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + out, err = c.addHasSBOMIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } else { + hsks, err := c.kv.Keys(ctx, hasSBOMCol) + if err != nil { + return nil, err + } + for _, hsk := range hsks { + link, err := byKeykv[*hasSBOMStruct](ctx, hasSBOMCol, hsk, c) + if err != nil { + return nil, err + } + out, err = c.addHasSBOMIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } + + return out, nil +} + +func (c *demoClient) addHasSBOMIfMatch(ctx context.Context, out []*model.HasSbom, + filter *model.HasSBOMSpec, link *hasSBOMStruct) ( + []*model.HasSbom, error) { + + if filter != nil { + if noMatch(filter.URI, link.URI) || + noMatch(toLower(filter.Algorithm), link.Algorithm) || + noMatch(toLower(filter.Digest), link.Digest) || + noMatch(filter.DownloadLocation, link.DownloadLocation) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) || + (filter.KnownSince != nil && filter.KnownSince.After(link.KnownSince)) { + return out, nil + } + // collect packages and artifacts from included software + pkgs, artifacts, err := c.getPackageVersionAndArtifacts(ctx, link.IncludedSoftware) + if err != nil { + return out, err + } + + pkgFilters, artFilters := getPackageAndArtifactFilters(filter.IncludedSoftware) + if !c.matchPackages(ctx, pkgFilters, pkgs) || !c.matchArtifacts(ctx, artFilters, artifacts) || + !c.matchDependencies(ctx, filter.IncludedDependencies, link.IncludedDependencies) || + !c.matchOccurrences(ctx, filter.IncludedOccurrences, link.IncludedOccurrences) { + return out, nil + } + + if filter.Subject != nil { + if filter.Subject.Package != nil { + if link.Pkg == "" { + return out, nil + } + p, err := c.buildPackageResponse(ctx, link.Pkg, filter.Subject.Package) + if err != nil { + return nil, err + } + if p == nil { + return out, nil + } + } else if filter.Subject.Artifact != nil { + if link.Artifact == "" { + return out, nil + } + if !c.artifactMatch(ctx, link.Artifact, filter.Subject.Artifact) { + return out, nil + } + } + } + } + sb, err := c.convHasSBOM(ctx, link) + if err != nil { + return nil, err + } + return append(out, sb), nil +} + +func getPackageAndArtifactFilters(filters []*model.PackageOrArtifactSpec) (pkgs []*model.PkgSpec, arts []*model.ArtifactSpec) { + for _, pkgOrArtSpec := range filters { + if pkgOrArtSpec.Package != nil { + pkgs = append(pkgs, pkgOrArtSpec.Package) + } else if pkgOrArtSpec.Artifact != nil { + arts = append(arts, pkgOrArtSpec.Artifact) + } + } + return +} diff --git a/pkg/assembler/backends/inmem/hasSBOM_test.go b/pkg/assembler/backends/keyvalue/hasSBOM_test.go similarity index 99% rename from pkg/assembler/backends/inmem/hasSBOM_test.go rename to pkg/assembler/backends/keyvalue/hasSBOM_test.go index 4cc4702925..5a89af72a4 100644 --- a/pkg/assembler/backends/inmem/hasSBOM_test.go +++ b/pkg/assembler/backends/keyvalue/hasSBOM_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -26,6 +26,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -252,11 +253,11 @@ var includedTestExpectedSBOM = &model.HasSbom{ Origin: "sbom_origin", Collector: "sbom_collector", IncludedSoftware: []model.PackageOrArtifact{ - includedTestExpectedPackage1, - includedTestExpectedPackage2, includedTestExpectedPackage3, includedTestExpectedArtifact1, includedTestExpectedArtifact2, + includedTestExpectedPackage1, + includedTestExpectedPackage2, }, IncludedDependencies: []*model.IsDependency{{ Package: includedTestExpectedPackage1, @@ -871,35 +872,6 @@ func TestHasSBOM(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InPkg: []*model.PkgInputSpec{p1}, - PkgArt: &model.PackageOrArtifactInputs{ - Packages: []*model.PkgInputSpec{p1}, - }, - Calls: []call{ - { - Sub: model.PackageOrArtifactInput{ - Package: p1, - }, - HS: &model.HasSBOMInputSpec{ - DownloadLocation: "location one", - }, - }, - { - Sub: model.PackageOrArtifactInput{ - Package: p1, - }, - HS: &model.HasSBOMInputSpec{ - DownloadLocation: "location two", - }, - }, - }, - Query: &model.HasSBOMSpec{ - ID: ptrfrom.String("-7"), - }, - ExpQueryErr: true, - }, { Name: "Query without hasSBOMSpec", InPkg: []*model.PkgInputSpec{p1}, @@ -2653,7 +2625,8 @@ func TestHasSBOM(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -2979,7 +2952,8 @@ func TestIngestHasSBOMs(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -3172,7 +3146,8 @@ func TestHasSBOMNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/hasSLSA.go b/pkg/assembler/backends/keyvalue/hasSLSA.go similarity index 56% rename from pkg/assembler/backends/inmem/hasSLSA.go rename to pkg/assembler/backends/keyvalue/hasSLSA.go index 38535954d0..d97ba977c8 100644 --- a/pkg/assembler/backends/inmem/hasSLSA.go +++ b/pkg/assembler/backends/keyvalue/hasSLSA.go @@ -13,56 +13,78 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" + "fmt" "slices" "sort" - "strconv" "strings" "time" - "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) type ( - hasSLSAList []*hasSLSAStruct hasSLSAStruct struct { - id uint32 - subject uint32 - builtFrom []uint32 - builtBy uint32 - buildType string - predicates []*model.SLSAPredicate - version string - start *time.Time - finish *time.Time - origin string - collector string + ThisID string + Subject string + BuiltFrom []string + BuiltBy string + BuildType string + Predicates []*model.SLSAPredicate + Version string + Start *time.Time + Finish *time.Time + Origin string + Collector string } ) -func (n *hasSLSAStruct) ID() uint32 { return n.id } +func (n *hasSLSAStruct) ID() string { return n.ThisID } +func (n *hasSLSAStruct) Key() string { + var st string + if n.Start != nil { + st = timeKey(*n.Start) + } + var fn string + if n.Finish != nil { + fn = timeKey(*n.Finish) + } + return strings.Join([]string{ + n.Subject, + fmt.Sprint(n.BuiltFrom), + n.BuiltBy, + n.BuildType, + fmt.Sprint(n.Predicates), + n.Version, + st, + fn, + n.Origin, + n.Collector, + }, ":") +} -func (n *hasSLSAStruct) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2+len(n.builtFrom)) +func (n *hasSLSAStruct) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 2+len(n.BuiltFrom)) if allowedEdges[model.EdgeHasSlsaSubject] { - out = append(out, n.subject) + out = append(out, n.Subject) } if allowedEdges[model.EdgeHasSlsaBuiltBy] { - out = append(out, n.builtBy) + out = append(out, n.BuiltBy) } if allowedEdges[model.EdgeHasSlsaMaterials] { - out = append(out, n.builtFrom...) + out = append(out, n.BuiltFrom...) } return out } -func (n *hasSLSAStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convSLSA(n) +func (n *hasSLSAStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convSLSA(ctx, n) } // Query HasSlsa @@ -72,25 +94,20 @@ func (c *demoClient) HasSlsa(ctx context.Context, filter *model.HasSLSASpec) ([] c.m.RLock() defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*hasSLSAStruct](id, c) + link, err := byIDkv[*hasSLSAStruct](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - hs, err := c.convSLSA(link) + hs, err := c.convSLSA(ctx, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.HasSlsa{hs}, nil } - var search []uint32 + var search []string foundOne := false var arts []*model.ArtifactSpec if filter != nil { @@ -99,24 +116,24 @@ func (c *demoClient) HasSlsa(ctx context.Context, filter *model.HasSLSASpec) ([] } for _, a := range arts { if !foundOne && a != nil { - exactArtifact, err := c.artifactExact(a) + exactArtifact, err := c.artifactExact(ctx, a) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.hasSLSAs...) + search = append(search, exactArtifact.HasSLSAs...) foundOne = true break } } } if !foundOne && filter != nil && filter.BuiltBy != nil { - exactBuilder, err := c.exactBuilder(filter.BuiltBy) + exactBuilder, err := c.exactBuilder(ctx, filter.BuiltBy) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactBuilder != nil { - search = append(search, exactBuilder.hasSLSAs...) + search = append(search, exactBuilder.HasSLSAs...) foundOne = true } } @@ -124,19 +141,26 @@ func (c *demoClient) HasSlsa(ctx context.Context, filter *model.HasSLSASpec) ([] var out []*model.HasSlsa if foundOne { for _, id := range search { - link, err := byID[*hasSLSAStruct](id, c) + link, err := byIDkv[*hasSLSAStruct](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addSLSAIfMatch(out, filter, link) + out, err = c.addSLSAIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.hasSLSAs { - var err error - out, err = c.addSLSAIfMatch(out, filter, link) + slsaKeys, err := c.kv.Keys(ctx, slsaCol) + if err != nil { + return nil, err + } + for _, slsak := range slsaKeys { + link, err := byKeykv[*hasSLSAStruct](ctx, slsaCol, slsak, c) + if err != nil { + return nil, err + } + out, err = c.addSLSAIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -183,50 +207,57 @@ func (c *demoClient) ingestSLSA(ctx context.Context, builtBy model.BuilderInputSpec, slsa model.SLSAInputSpec, readOnly bool) ( *model.HasSlsa, error, ) { + preds := convSLSAP(slsa.SlsaPredicate) + in := &hasSLSAStruct{ + BuildType: slsa.BuildType, + Predicates: preds, + Version: slsa.SlsaVersion, + Origin: slsa.Origin, + Collector: slsa.Collector, + } + if slsa.StartedOn != nil { + t := slsa.StartedOn.UTC() + in.Start = &t + } + if slsa.FinishedOn != nil { + t := slsa.FinishedOn.UTC() + in.Finish = &t + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - s, err := c.artifactByKey(subject.Algorithm, subject.Digest) + s, err := c.artifactByInput(ctx, &subject) if err != nil { return nil, gqlerror.Errorf("IngestSLSA :: Subject artifact not found") } + in.Subject = s.ThisID + var bfs []*artStruct - var bfIDs []uint32 + var bfIDs []string for i, a := range builtFrom { - b, err := c.artifactByKey(a.Algorithm, a.Digest) + b, err := c.artifactByInput(ctx, a) if err != nil { return nil, gqlerror.Errorf("IngestSLSA :: BuiltFrom %d artifact not found", i) } bfs = append(bfs, b) - bfIDs = append(bfIDs, b.id) + bfIDs = append(bfIDs, b.ID()) } slices.Sort(bfIDs) + in.BuiltFrom = bfIDs - b, err := c.builderByKey(builtBy.URI) + b, err := c.builderByInput(ctx, &builtBy) if err != nil { return nil, gqlerror.Errorf("IngestSLSA :: Builder not found") } + in.BuiltBy = b.ThisID - preds := convSLSAP(slsa.SlsaPredicate) - - // Just picking the first builtFrom found to search the backedges - for _, slID := range bfs[0].hasSLSAs { - sl, err := byID[*hasSLSAStruct](slID, c) - if err != nil { - return nil, gqlerror.Errorf("IngestSLSA :: Internal db error, bad backedge") - } - if sl.subject == s.id && - slices.Equal(sl.builtFrom, bfIDs) && - sl.builtBy == b.id && - sl.buildType == slsa.BuildType && - cmp.Equal(sl.predicates, preds) && - sl.version == slsa.SlsaVersion && - timePtrEqual(sl.start, slsa.StartedOn) && - timePtrEqual(sl.finish, slsa.FinishedOn) && - sl.origin == slsa.Origin && - sl.collector == slsa.Collector { - return c.convSLSA(sl) - } + out, err := byKeykv[*hasSLSAStruct](ctx, slsaCol, in.Key(), c) + if err == nil { + return c.convSLSA(ctx, out) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } if readOnly { @@ -236,28 +267,27 @@ func (c *demoClient) ingestSLSA(ctx context.Context, return s, err } - sl := &hasSLSAStruct{ - id: c.getNextID(), - subject: s.id, - builtFrom: bfIDs, - builtBy: b.id, - buildType: slsa.BuildType, - predicates: preds, - version: slsa.SlsaVersion, - start: slsa.StartedOn, - finish: slsa.FinishedOn, - origin: slsa.Origin, - collector: slsa.Collector, - } - c.index[sl.id] = sl - c.hasSLSAs = append(c.hasSLSAs, sl) - s.setHasSLSAs(sl.id) + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, slsaCol, in); err != nil { + return nil, err + } + if err := s.setHasSLSAs(ctx, in.ThisID, c); err != nil { + return nil, err + } for _, a := range bfs { - a.setHasSLSAs(sl.id) + if err := a.setHasSLSAs(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := b.setHasSLSAs(ctx, in.ThisID, c); err != nil { + return nil, err + } + if err := setkv(ctx, slsaCol, in, c); err != nil { + return nil, err } - b.setHasSLSAs(sl.id) - return c.convSLSA(sl) + return c.convSLSA(ctx, in) } func convSLSAP(in []*model.SLSAPredicateInputSpec) []*model.SLSAPredicate { @@ -272,63 +302,63 @@ func convSLSAP(in []*model.SLSAPredicateInputSpec) []*model.SLSAPredicate { return rv } -func (c *demoClient) convSLSA(in *hasSLSAStruct) (*model.HasSlsa, error) { - sub, err := byID[*artStruct](in.subject, c) +func (c *demoClient) convSLSA(ctx context.Context, in *hasSLSAStruct) (*model.HasSlsa, error) { + sub, err := byIDkv[*artStruct](ctx, in.Subject, c) if err != nil { return nil, err } var bfs []*model.Artifact - for _, id := range in.builtFrom { - a, err := byID[*artStruct](id, c) + for _, id := range in.BuiltFrom { + a, err := byIDkv[*artStruct](ctx, id, c) if err != nil { return nil, err } bfs = append(bfs, c.convArtifact(a)) } - bb, err := byID[*builderStruct](in.builtBy, c) + bb, err := byIDkv[*builderStruct](ctx, in.BuiltBy, c) if err != nil { return nil, err } return &model.HasSlsa{ - ID: nodeID(in.id), + ID: in.ThisID, Subject: c.convArtifact(sub), Slsa: &model.Slsa{ BuiltFrom: bfs, BuiltBy: c.convBuilder(bb), - BuildType: in.buildType, - SlsaPredicate: in.predicates, - SlsaVersion: in.version, - StartedOn: in.start, - FinishedOn: in.finish, - Origin: in.origin, - Collector: in.collector, + BuildType: in.BuildType, + SlsaPredicate: in.Predicates, + SlsaVersion: in.Version, + StartedOn: in.Start, + FinishedOn: in.Finish, + Origin: in.Origin, + Collector: in.Collector, }, }, nil } -func (c *demoClient) addSLSAIfMatch(out []*model.HasSlsa, +func (c *demoClient) addSLSAIfMatch(ctx context.Context, out []*model.HasSlsa, filter *model.HasSLSASpec, link *hasSLSAStruct) ( []*model.HasSlsa, error, ) { - bb, err := byID[*builderStruct](link.builtBy, c) + bb, err := byIDkv[*builderStruct](ctx, link.BuiltBy, c) if err != nil { return nil, err } - if noMatch(filter.BuildType, link.buildType) || - noMatch(filter.SlsaVersion, link.version) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) || - (filter.StartedOn != nil && (link.start == nil || !filter.StartedOn.Equal(*link.start))) || - (filter.FinishedOn != nil && (link.finish == nil || !filter.FinishedOn.Equal(*link.finish))) || - (filter.BuiltBy != nil && filter.BuiltBy.ID != nil && *filter.BuiltBy.ID != nodeID(bb.id)) || - (filter.BuiltBy != nil && filter.BuiltBy.URI != nil && *filter.BuiltBy.URI != bb.uri) || - !matchSLSAPreds(link.predicates, filter.Predicate) || - !c.matchArtifacts([]*model.ArtifactSpec{filter.Subject}, []uint32{link.subject}) || - !c.matchArtifacts(filter.BuiltFrom, link.builtFrom) { + if noMatch(filter.BuildType, link.BuildType) || + noMatch(filter.SlsaVersion, link.Version) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) || + (filter.StartedOn != nil && (link.Start == nil || !filter.StartedOn.Equal(*link.Start))) || + (filter.FinishedOn != nil && (link.Finish == nil || !filter.FinishedOn.Equal(*link.Finish))) || + (filter.BuiltBy != nil && filter.BuiltBy.ID != nil && *filter.BuiltBy.ID != bb.ThisID) || + (filter.BuiltBy != nil && filter.BuiltBy.URI != nil && *filter.BuiltBy.URI != bb.URI) || + !matchSLSAPreds(link.Predicates, filter.Predicate) || + !c.matchArtifacts(ctx, []*model.ArtifactSpec{filter.Subject}, []string{link.Subject}) || + !c.matchArtifacts(ctx, filter.BuiltFrom, link.BuiltFrom) { return out, nil } - hs, err := c.convSLSA(link) + hs, err := c.convSLSA(ctx, link) if err != nil { return nil, err } diff --git a/pkg/assembler/backends/inmem/hasSLSA_test.go b/pkg/assembler/backends/keyvalue/hasSLSA_test.go similarity index 96% rename from pkg/assembler/backends/inmem/hasSLSA_test.go rename to pkg/assembler/backends/keyvalue/hasSLSA_test.go index cf1256eee0..ca1f452bb2 100644 --- a/pkg/assembler/backends/inmem/hasSLSA_test.go +++ b/pkg/assembler/backends/keyvalue/hasSLSA_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -298,14 +299,14 @@ func TestHasSLSA(t *testing.T) { Subject: a1out, Slsa: &model.Slsa{ BuiltBy: b1out, - BuiltFrom: []*model.Artifact{a2out}, + BuiltFrom: []*model.Artifact{a2out, a3out}, }, }, { Subject: a1out, Slsa: &model.Slsa{ BuiltBy: b1out, - BuiltFrom: []*model.Artifact{a2out, a3out}, + BuiltFrom: []*model.Artifact{a2out}, }, }, }, @@ -468,29 +469,6 @@ func TestHasSLSA(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InArt: []*model.ArtifactInputSpec{a1, a2}, - InBld: []*model.BuilderInputSpec{b1, b2}, - Calls: []call{ - { - Sub: a1, - BF: []*model.ArtifactInputSpec{a2}, - BB: b1, - SLSA: &model.SLSAInputSpec{}, - }, - { - Sub: a1, - BF: []*model.ArtifactInputSpec{a2}, - BB: b2, - SLSA: &model.SLSAInputSpec{}, - }, - }, - Query: &model.HasSLSASpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -498,7 +476,8 @@ func TestHasSLSA(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -704,14 +683,14 @@ func TestIngestHasSLSAs(t *testing.T) { Subject: a1out, Slsa: &model.Slsa{ BuiltBy: b1out, - BuiltFrom: []*model.Artifact{a2out}, + BuiltFrom: []*model.Artifact{a2out, a3out}, }, }, { Subject: a1out, Slsa: &model.Slsa{ BuiltBy: b1out, - BuiltFrom: []*model.Artifact{a2out, a3out}, + BuiltFrom: []*model.Artifact{a2out}, }, }, }, @@ -723,7 +702,8 @@ func TestIngestHasSLSAs(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -831,7 +811,8 @@ func TestHasSLSANeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/hasSourceAt.go b/pkg/assembler/backends/keyvalue/hasSourceAt.go similarity index 54% rename from pkg/assembler/backends/inmem/hasSourceAt.go rename to pkg/assembler/backends/keyvalue/hasSourceAt.go index e733715ed3..69fa780d47 100644 --- a/pkg/assembler/backends/inmem/hasSourceAt.go +++ b/pkg/assembler/backends/keyvalue/hasSourceAt.go @@ -13,45 +13,56 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link between sources and packages (HasSourceAt) -type hasSrcList []*srcMapLink type srcMapLink struct { - id uint32 - sourceID uint32 - packageID uint32 - knownSince time.Time - justification string - origin string - collector string + ThisID string + SourceID string + PackageID string + KnownSince time.Time + Justification string + Origin string + Collector string } -func (n *srcMapLink) ID() uint32 { return n.id } +func (n *srcMapLink) ID() string { return n.ThisID } +func (n *srcMapLink) Key() string { + return strings.Join([]string{ + n.SourceID, + n.PackageID, + timeKey(n.KnownSince), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *srcMapLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2) +func (n *srcMapLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 2) if allowedEdges[model.EdgeHasSourceAtPackage] { - out = append(out, n.packageID) + out = append(out, n.PackageID) } if allowedEdges[model.EdgeHasSourceAtSource] { - out = append(out, n.sourceID) + out = append(out, n.SourceID) } return out } -func (n *srcMapLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildHasSourceAt(n, nil, true) +func (n *srcMapLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildHasSourceAt(ctx, n, nil, true) } // Ingest HasSourceAt @@ -75,81 +86,61 @@ func (c *demoClient) IngestHasSourceAt(ctx context.Context, packageArg model.Pkg func (c *demoClient) ingestHasSourceAt(ctx context.Context, packageArg model.PkgInputSpec, pkgMatchType model.MatchFlags, source model.SourceInputSpec, hasSourceAt model.HasSourceAtInputSpec, readOnly bool) (*model.HasSourceAt, error) { funcName := "IngestHasSourceAt" + + in := &srcMapLink{ + KnownSince: hasSourceAt.KnownSince.UTC(), + Justification: hasSourceAt.Justification, + Origin: hasSourceAt.Origin, + Collector: hasSourceAt.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - sourceID, err := getSourceIDFromInput(c, source) + srcName, err := c.getSourceNameFromInput(ctx, source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - srcName, err := byID[*srcNameNode](sourceID, c) + in.SourceID = srcName.ThisID + + pkgNameOrVersionNode, err := c.getPackageNameOrVerFromInput(ctx, packageArg, pkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - sourceHasSourceLinks := srcName.srcMapLinks + in.PackageID = pkgNameOrVersionNode.ID() - packageID, err := getPackageIDFromInput(c, packageArg, pkgMatchType) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + out, err := byKeykv[*srcMapLink](ctx, hsaCol, in.Key(), c) + if err == nil { + return c.buildHasSourceAt(ctx, out, nil, true) } - pkgNameOrVersionNode, err := byID[pkgNameOrVersion](packageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + if !errors.Is(err, kv.NotFoundError) { + return nil, err } - packageHasSourceLinks := pkgNameOrVersionNode.getSrcMapLinks() - var searchIDs []uint32 - if len(packageHasSourceLinks) < len(sourceHasSourceLinks) { - searchIDs = packageHasSourceLinks - } else { - searchIDs = sourceHasSourceLinks + if readOnly { + c.m.RUnlock() + s, err := c.ingestHasSourceAt(ctx, packageArg, pkgMatchType, source, hasSourceAt, false) + c.m.RLock() // relock so that defer unlock does not panic + return s, err } - // Don't insert duplicates - duplicate := false - collectedSrcMapLink := srcMapLink{} - for _, id := range searchIDs { - v, err := byID[*srcMapLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if packageID == v.packageID && sourceID == v.sourceID && hasSourceAt.Justification == v.justification && - hasSourceAt.Origin == v.origin && hasSourceAt.Collector == v.collector && hasSourceAt.KnownSince.Equal(v.knownSince) { - collectedSrcMapLink = *v - duplicate = true - break - } + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, hsaCol, in); err != nil { + return nil, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - s, err := c.ingestHasSourceAt(ctx, packageArg, pkgMatchType, source, hasSourceAt, false) - c.m.RLock() // relock so that defer unlock does not panic - return s, err - } - // store the link - collectedSrcMapLink = srcMapLink{ - id: c.getNextID(), - sourceID: sourceID, - packageID: packageID, - knownSince: hasSourceAt.KnownSince.UTC(), - justification: hasSourceAt.Justification, - origin: hasSourceAt.Origin, - collector: hasSourceAt.Collector, - } - c.index[collectedSrcMapLink.id] = &collectedSrcMapLink - c.hasSources = append(c.hasSources, &collectedSrcMapLink) - // set the backlinks - pkgNameOrVersionNode.setSrcMapLinks(collectedSrcMapLink.id) - srcName.setSrcMapLinks(collectedSrcMapLink.id) + // set the backlinks + if err := pkgNameOrVersionNode.setSrcMapLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - - // build return GraphQL type - foundHasSourceAt, err := c.buildHasSourceAt(&collectedSrcMapLink, nil, true) - if err != nil { + if err := srcName.setSrcMapLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + if err := setkv(ctx, hsaCol, in, c); err != nil { return nil, err } - return foundHasSourceAt, nil + + return c.buildHasSourceAt(ctx, in, nil, true) } // Query HasSourceAt @@ -158,17 +149,12 @@ func (c *demoClient) HasSourceAt(ctx context.Context, filter *model.HasSourceAtS defer c.m.RUnlock() funcName := "HasSourceAt" if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*srcMapLink](id, c) + link, err := byIDkv[*srcMapLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - foundHasSourceAt, err := c.buildHasSourceAt(link, filter, true) + foundHasSourceAt, err := c.buildHasSourceAt(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -177,15 +163,15 @@ func (c *demoClient) HasSourceAt(ctx context.Context, filter *model.HasSourceAtS // Cant really search for an exact Pkg, as these can be linked to either // names or versions, only search Source backedges. - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Source != nil { - exactSource, err := c.exactSource(filter.Source) + exactSource, err := c.exactSource(ctx, filter.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.srcMapLinks...) + search = append(search, exactSource.SrcMapLinks...) foundOne = true } } @@ -193,19 +179,26 @@ func (c *demoClient) HasSourceAt(ctx context.Context, filter *model.HasSourceAtS var out []*model.HasSourceAt if foundOne { for _, id := range search { - link, err := byID[*srcMapLink](id, c) + link, err := byIDkv[*srcMapLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addSrcIfMatch(out, filter, link) + out, err = c.addSrcIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.hasSources { - var err error - out, err = c.addSrcIfMatch(out, filter, link) + hsaKeys, err := c.kv.Keys(ctx, hsaCol) + if err != nil { + return nil, err + } + for _, hsak := range hsaKeys { + link, err := byKeykv[*srcMapLink](ctx, hsaCol, hsak, c) + if err != nil { + return nil, err + } + out, err = c.addSrcIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -215,25 +208,25 @@ func (c *demoClient) HasSourceAt(ctx context.Context, filter *model.HasSourceAtS return out, nil } -func (c *demoClient) buildHasSourceAt(link *srcMapLink, filter *model.HasSourceAtSpec, ingestOrIDProvided bool) (*model.HasSourceAt, error) { +func (c *demoClient) buildHasSourceAt(ctx context.Context, link *srcMapLink, filter *model.HasSourceAtSpec, ingestOrIDProvided bool) (*model.HasSourceAt, error) { var p *model.Package var s *model.Source var err error if filter != nil { - p, err = c.buildPackageResponse(link.packageID, filter.Package) + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Package) if err != nil { return nil, err } - s, err = c.buildSourceResponse(link.sourceID, filter.Source) + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Source) if err != nil { return nil, err } } else { - p, err = c.buildPackageResponse(link.packageID, nil) + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } - s, err = c.buildSourceResponse(link.sourceID, nil) + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -252,33 +245,33 @@ func (c *demoClient) buildHasSourceAt(link *srcMapLink, filter *model.HasSourceA } newHSA := model.HasSourceAt{ - ID: nodeID(link.id), + ID: link.ThisID, Package: p, Source: s, - KnownSince: link.knownSince, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, + KnownSince: link.KnownSince, + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, } return &newHSA, nil } -func (c *demoClient) addSrcIfMatch(out []*model.HasSourceAt, +func (c *demoClient) addSrcIfMatch(ctx context.Context, out []*model.HasSourceAt, filter *model.HasSourceAtSpec, link *srcMapLink) ( []*model.HasSourceAt, error) { - if filter != nil && noMatch(filter.Justification, link.justification) { + if filter != nil && noMatch(filter.Justification, link.Justification) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - if filter != nil && filter.KnownSince != nil && !filter.KnownSince.Equal(link.knownSince) { + if filter != nil && filter.KnownSince != nil && !filter.KnownSince.Equal(link.KnownSince) { return out, nil } - foundHasSourceAt, err := c.buildHasSourceAt(link, filter, false) + foundHasSourceAt, err := c.buildHasSourceAt(ctx, link, filter, false) if err != nil { return nil, err } diff --git a/pkg/assembler/backends/inmem/hasSourceAt_test.go b/pkg/assembler/backends/keyvalue/hasSourceAt_test.go similarity index 97% rename from pkg/assembler/backends/inmem/hasSourceAt_test.go rename to pkg/assembler/backends/keyvalue/hasSourceAt_test.go index 7227c6c7db..d535aec3c6 100644 --- a/pkg/assembler/backends/inmem/hasSourceAt_test.go +++ b/pkg/assembler/backends/keyvalue/hasSourceAt_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -436,11 +437,11 @@ func TestHasSourceAt(t *testing.T) { }, ExpHSA: []*model.HasSourceAt{ { - Package: p1out, + Package: p1outName, Source: s1out, }, { - Package: p1outName, + Package: p1out, Source: s1out, }, }, @@ -475,25 +476,6 @@ func TestHasSourceAt(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InPkg: []*model.PkgInputSpec{p1}, - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Pkg: p1, - Src: s1, - Match: &model.MatchFlags{ - Pkg: model.PkgMatchTypeSpecificVersion, - }, - HSA: &model.HasSourceAtInputSpec{}, - }, - }, - Query: &model.HasSourceAtSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -501,7 +483,8 @@ func TestHasSourceAt(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -753,7 +736,8 @@ func TestIngestHasSourceAts(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -862,7 +846,8 @@ func TestHasSourceAtNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/hashEqual.go b/pkg/assembler/backends/keyvalue/hashEqual.go similarity index 55% rename from pkg/assembler/backends/inmem/hashEqual.go rename to pkg/assembler/backends/keyvalue/hashEqual.go index 7b0e311283..6233d0f626 100644 --- a/pkg/assembler/backends/inmem/hashEqual.go +++ b/pkg/assembler/backends/keyvalue/hashEqual.go @@ -13,66 +13,51 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" "fmt" "slices" - "strconv" "strings" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) // Internal hashEqual -type ( - hashEqualList []*hashEqualStruct - hashEqualStruct struct { - id uint32 - artifacts []uint32 - justification string - origin string - collector string - } -) +type hashEqualStruct struct { + ThisID string + Artifacts []string + Justification string + Origin string + Collector string +} -func (n *hashEqualStruct) ID() uint32 { return n.id } +func (n *hashEqualStruct) ID() string { return n.ThisID } +func (n *hashEqualStruct) Key() string { + return strings.Join([]string{ + fmt.Sprint(n.Artifacts), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *hashEqualStruct) Neighbors(allowedEdges edgeMap) []uint32 { +func (n *hashEqualStruct) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgeHashEqualArtifact] { - return n.artifacts + return n.Artifacts } - return []uint32{} + return []string{} } -func (n *hashEqualStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convHashEqual(n) +func (n *hashEqualStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convHashEqual(ctx, n) } -// TODO convert to unit tests -// func registerAllHashEqual(client *demoClient) { -// strings.ToLower(string(checksum.Algorithm)) + ":" + checksum.Value -// -client.registerHashEqual([]*model.Artifact{client.artifacts[0], client.artifacts[1], client.artifacts[2]}, "different algorithm for the same artifact", "inmem backend", "inmem backend") -// client.IngestHashEqual( -// context.Background(), -// model.ArtifactInputSpec{ -// Digest: "7A8F47318E4676DACB0142AFA0B83029CD7BEFD9", -// Algorithm: "sha1", -// }, -// model.ArtifactInputSpec{ -// Digest: "6bbb0da1891646e58eb3e6a63af3a6fc3c8eb5a0d44824cba581d2e14a0450cf", -// Algorithm: "sha256", -// }, -// model.HashEqualInputSpec{ -// Justification: "these two are the same", -// Origin: "inmem backend", -// Collector: "inmem backend", -// }) -// } - // Ingest HashEqual func (c *demoClient) IngestHashEquals(ctx context.Context, artifacts []*model.ArtifactInputSpec, otherArtifacts []*model.ArtifactInputSpec, hashEquals []*model.HashEqualInputSpec) ([]*model.HashEqual, error) { @@ -92,36 +77,33 @@ func (c *demoClient) IngestHashEqual(ctx context.Context, artifact model.Artifac } func (c *demoClient) ingestHashEqual(ctx context.Context, artifact model.ArtifactInputSpec, equalArtifact model.ArtifactInputSpec, hashEqual model.HashEqualInputSpec, readOnly bool) (*model.HashEqual, error) { + in := &hashEqualStruct{ + Justification: hashEqual.Justification, + Origin: hashEqual.Origin, + Collector: hashEqual.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - aInt1, err := c.artifactByKey(artifact.Algorithm, artifact.Digest) + aInt1, err := c.artifactByInput(ctx, &artifact) if err != nil { return nil, gqlerror.Errorf("IngestHashEqual :: Artifact not found") } - aInt2, err := c.artifactByKey(equalArtifact.Algorithm, equalArtifact.Digest) + aInt2, err := c.artifactByInput(ctx, &equalArtifact) if err != nil { return nil, gqlerror.Errorf("IngestHashEqual :: Artifact not found") } - artIDs := []uint32{aInt1.id, aInt2.id} + artIDs := []string{aInt1.ThisID, aInt2.ThisID} slices.Sort(artIDs) + in.Artifacts = artIDs - // Search backedges for existing. - searchHEs := slices.Clone(aInt1.hashEquals) - searchHEs = append(searchHEs, aInt2.hashEquals...) - - for _, he := range searchHEs { - h, err := byID[*hashEqualStruct](he, c) - if err != nil { - return nil, gqlerror.Errorf( - "IngestHashEqual :: Bad hashEqual id stored on existing artifact: %s", err) - } - if h.justification == hashEqual.Justification && - h.origin == hashEqual.Origin && - h.collector == hashEqual.Collector && - slices.Equal(h.artifacts, artIDs) { - return c.convHashEqual(h) - } + out, err := byKeykv[*hashEqualStruct](ctx, hashEqCol, in.Key(), c) + if err == nil { + return c.convHashEqual(ctx, out) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } if readOnly { @@ -131,33 +113,35 @@ func (c *demoClient) ingestHashEqual(ctx context.Context, artifact model.Artifac return he, err } - he := &hashEqualStruct{ - id: c.getNextID(), - artifacts: artIDs, - justification: hashEqual.Justification, - origin: hashEqual.Origin, - collector: hashEqual.Collector, + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, hashEqCol, in); err != nil { + return nil, err + } + if err := aInt1.setHashEquals(ctx, in.ThisID, c); err != nil { + return nil, err + } + if err := aInt2.setHashEquals(ctx, in.ThisID, c); err != nil { + return nil, err + } + if err := setkv(ctx, hashEqCol, in, c); err != nil { + return nil, err } - c.index[he.id] = he - c.hashEquals = append(c.hashEquals, he) - aInt1.setHashEquals(he.id) - aInt2.setHashEquals(he.id) - return c.convHashEqual(he) + return c.convHashEqual(ctx, in) } -func (c *demoClient) matchArtifacts(filter []*model.ArtifactSpec, value []uint32) bool { +func (c *demoClient) matchArtifacts(ctx context.Context, filter []*model.ArtifactSpec, value []string) bool { val := slices.Clone(value) - var matchID []uint32 + var matchID []string var matchPartial []*model.ArtifactSpec for _, aSpec := range filter { if aSpec == nil { continue } - a, _ := c.artifactExact(aSpec) + a, _ := c.artifactExact(ctx, aSpec) // drop error here if ID is bad if a != nil { - matchID = append(matchID, a.id) + matchID = append(matchID, a.ID()) } else if aSpec.ID != nil { // We had an id but it didn't match return false @@ -175,12 +159,12 @@ func (c *demoClient) matchArtifacts(filter []*model.ArtifactSpec, value []uint32 match := false remove := -1 for i, v := range val { - a, err := byID[*artStruct](v, c) + a, err := byIDkv[*artStruct](ctx, v, c) if err != nil { return false } - if (m.Algorithm == nil || strings.ToLower(*m.Algorithm) == a.algorithm) && - (m.Digest == nil || strings.ToLower(*m.Digest) == a.digest) { + if (m.Algorithm == nil || strings.ToLower(*m.Algorithm) == a.Algorithm) && + (m.Digest == nil || strings.ToLower(*m.Digest) == a.Digest) { match = true remove = i break @@ -201,34 +185,29 @@ func (c *demoClient) HashEqual(ctx context.Context, filter *model.HashEqualSpec) c.m.RLock() defer c.m.RUnlock() if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*hashEqualStruct](id, c) + link, err := byIDkv[*hashEqualStruct](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - he, err := c.convHashEqual(link) + he, err := c.convHashEqual(ctx, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.HashEqual{he}, nil } - var search []uint32 + var search []string foundOne := false for _, a := range filter.Artifacts { if !foundOne && a != nil { - exactArtifact, err := c.artifactExact(a) + exactArtifact, err := c.artifactExact(ctx, a) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.hashEquals...) + search = append(search, exactArtifact.HashEquals...) foundOne = true break } @@ -238,19 +217,26 @@ func (c *demoClient) HashEqual(ctx context.Context, filter *model.HashEqualSpec) var out []*model.HashEqual if foundOne { for _, id := range search { - link, err := byID[*hashEqualStruct](id, c) + link, err := byIDkv[*hashEqualStruct](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addHEIfMatch(out, filter, link) + out, err = c.addHEIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.hashEquals { - var err error - out, err = c.addHEIfMatch(out, filter, link) + heKeys, err := c.kv.Keys(ctx, hashEqCol) + if err != nil { + return nil, err + } + for _, hek := range heKeys { + link, err := byKeykv[*hashEqualStruct](ctx, hashEqCol, hek, c) + if err != nil { + return nil, err + } + out, err = c.addHEIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -259,35 +245,35 @@ func (c *demoClient) HashEqual(ctx context.Context, filter *model.HashEqualSpec) return out, nil } -func (c *demoClient) convHashEqual(h *hashEqualStruct) (*model.HashEqual, error) { +func (c *demoClient) convHashEqual(ctx context.Context, h *hashEqualStruct) (*model.HashEqual, error) { var artifacts []*model.Artifact - for _, id := range h.artifacts { - a, err := byID[*artStruct](id, c) + for _, id := range h.Artifacts { + a, err := byIDkv[*artStruct](ctx, id, c) if err != nil { return nil, fmt.Errorf("convHashEqual: struct contains bad artifact id") } artifacts = append(artifacts, c.convArtifact(a)) } return &model.HashEqual{ - ID: nodeID(h.id), - Justification: h.justification, + ID: h.ThisID, + Justification: h.Justification, Artifacts: artifacts, - Origin: h.origin, - Collector: h.collector, + Origin: h.Origin, + Collector: h.Collector, }, nil } -func (c *demoClient) addHEIfMatch(out []*model.HashEqual, +func (c *demoClient) addHEIfMatch(ctx context.Context, out []*model.HashEqual, filter *model.HashEqualSpec, link *hashEqualStruct) ( []*model.HashEqual, error, ) { - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) || - !c.matchArtifacts(filter.Artifacts, link.artifacts) { + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) || + !c.matchArtifacts(ctx, filter.Artifacts, link.Artifacts) { return out, nil } - he, err := c.convHashEqual(link) + he, err := c.convHashEqual(ctx, link) if err != nil { return nil, err } diff --git a/pkg/assembler/backends/inmem/hashEqual_test.go b/pkg/assembler/backends/keyvalue/hashEqual_test.go similarity index 96% rename from pkg/assembler/backends/inmem/hashEqual_test.go rename to pkg/assembler/backends/keyvalue/hashEqual_test.go index 21fd856628..5c77b50bef 100644 --- a/pkg/assembler/backends/inmem/hashEqual_test.go +++ b/pkg/assembler/backends/keyvalue/hashEqual_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -383,31 +384,6 @@ func TestHashEqual(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InArt: []*model.ArtifactInputSpec{a1, a2, a3}, - Calls: []call{ - { - A1: a1, - A2: a2, - HE: &model.HashEqualInputSpec{}, - }, - { - A1: a2, - A2: a3, - HE: &model.HashEqualInputSpec{}, - }, - { - A1: a1, - A2: a3, - HE: &model.HashEqualInputSpec{}, - }, - }, - Query: &model.HashEqualSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -415,7 +391,8 @@ func TestHashEqual(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -684,7 +661,8 @@ func TestIngestHashEquals(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -784,7 +762,8 @@ func TestHashEqualNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/helpers_test.go b/pkg/assembler/backends/keyvalue/helpers_test.go similarity index 97% rename from pkg/assembler/backends/inmem/helpers_test.go rename to pkg/assembler/backends/keyvalue/helpers_test.go index fb2a8561fb..06a00f025e 100644 --- a/pkg/assembler/backends/inmem/helpers_test.go +++ b/pkg/assembler/backends/keyvalue/helpers_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "cmp" @@ -25,7 +25,7 @@ import ( // helper functions to canonically compare nodes // this file is a duplicate of testhelpers_test.go -- pkg_test.go is in -// the inmem package, so it cannot use the definitions in testhelpers_test.go +// the keyvalue package, so it cannot use the definitions in testhelpers_test.go ////////////////////// comparison functions ////////////////////// diff --git a/pkg/assembler/backends/inmem/isDependency.go b/pkg/assembler/backends/keyvalue/isDependency.go similarity index 54% rename from pkg/assembler/backends/inmem/isDependency.go rename to pkg/assembler/backends/keyvalue/isDependency.go index a12e5bef63..b64cae767c 100644 --- a/pkg/assembler/backends/inmem/isDependency.go +++ b/pkg/assembler/backends/keyvalue/isDependency.go @@ -13,41 +13,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link between packages and dependent packages (isDependency) -type isDependencyList []*isDependencyLink type isDependencyLink struct { - id uint32 - packageID uint32 - depPackageID uint32 - versionRange string - dependencyType model.DependencyType - justification string - origin string - collector string + ThisID string + PackageID string + DepPackageID string + VersionRange string + DependencyType model.DependencyType + Justification string + Origin string + Collector string } -func (n *isDependencyLink) ID() uint32 { return n.id } +func (n *isDependencyLink) ID() string { return n.ThisID } +func (n *isDependencyLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.DepPackageID, + n.VersionRange, + string(n.DependencyType), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *isDependencyLink) Neighbors(allowedEdges edgeMap) []uint32 { +func (n *isDependencyLink) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgeIsDependencyPackage] { - return []uint32{n.packageID, n.depPackageID} + return []string{n.PackageID, n.DepPackageID} } - return []uint32{} + return []string{} } -func (n *isDependencyLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildIsDependency(n, nil, true) +func (n *isDependencyLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildIsDependency(ctx, n, nil, true) } // Ingest IngestDependencies @@ -73,88 +85,64 @@ func (c *demoClient) IngestDependency(ctx context.Context, packageArg model.PkgI func (c *demoClient) ingestDependency(ctx context.Context, packageArg model.PkgInputSpec, dependentPackageArg model.PkgInputSpec, depPkgMatchType model.MatchFlags, dependency model.IsDependencyInputSpec, readOnly bool) (*model.IsDependency, error) { funcName := "IngestDependency" + + inLink := &isDependencyLink{ + VersionRange: dependency.VersionRange, + DependencyType: dependency.DependencyType, + Justification: dependency.Justification, + Origin: dependency.Origin, + Collector: dependency.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) // for IsDependency the dependent package will return the ID at the // packageName node. VersionRange will be used to specify the versions are // the attestation relates to - packageID, err := getPackageIDFromInput(c, packageArg, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}) + foundPkgVersion, err := c.getPackageVerFromInput(ctx, packageArg) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundPkgVersion, err := byID[*pkgVersionNode](packageID, c) + inLink.PackageID = foundPkgVersion.ThisID + + depPkg, err := c.getPackageNameOrVerFromInput(ctx, dependentPackageArg, depPkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - packageDependencies := foundPkgVersion.isDependencyLinks + inLink.DepPackageID = depPkg.ID() - depPackageID, err := getPackageIDFromInput(c, dependentPackageArg, depPkgMatchType) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + outLink, err := byKeykv[*isDependencyLink](ctx, isDepCol, inLink.Key(), c) + if err == nil { + return c.buildIsDependency(ctx, outLink, nil, true) } - depPkg, err := byID[pkgNameOrVersion](depPackageID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) + if !errors.Is(err, kv.NotFoundError) { + return nil, err } - depPackageDependencies := depPkg.getIsDependencyLinks() - var searchIDs []uint32 - if len(packageDependencies) < len(depPackageDependencies) { - searchIDs = packageDependencies - } else { - searchIDs = depPackageDependencies + if readOnly { + c.m.RUnlock() + d, err := c.ingestDependency(ctx, packageArg, dependentPackageArg, depPkgMatchType, dependency, false) + c.m.RLock() // relock so that defer unlock does not panic + return d, err } - // Don't insert duplicates - duplicate := false - collectedIsDependencyLink := isDependencyLink{} - for _, id := range searchIDs { - v, err := byID[*isDependencyLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - if packageID == v.packageID && depPackageID == v.depPackageID && dependency.Justification == v.justification && - dependency.Origin == v.origin && dependency.Collector == v.collector && - dependency.VersionRange == v.versionRange && dependency.DependencyType == v.dependencyType { - - collectedIsDependencyLink = *v - duplicate = true - break - } + inLink.ThisID = c.getNextID() + if err := c.addToIndex(ctx, isDepCol, inLink); err != nil { + return nil, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - d, err := c.ingestDependency(ctx, packageArg, dependentPackageArg, depPkgMatchType, dependency, false) - c.m.RLock() // relock so that defer unlock does not panic - return d, err - } - // store the link - collectedIsDependencyLink = isDependencyLink{ - id: c.getNextID(), - packageID: packageID, - depPackageID: depPackageID, - versionRange: dependency.VersionRange, - dependencyType: dependency.DependencyType, - justification: dependency.Justification, - origin: dependency.Origin, - collector: dependency.Collector, - } - c.index[collectedIsDependencyLink.id] = &collectedIsDependencyLink - c.isDependencies = append(c.isDependencies, &collectedIsDependencyLink) - // set the backlinks - foundPkgVersion.setIsDependencyLinks(collectedIsDependencyLink.id) - depPkg.setIsDependencyLinks(collectedIsDependencyLink.id) + if err := foundPkgVersion.setIsDependencyLinks(ctx, inLink.ThisID, c); err != nil { + return nil, err } - - // build return GraphQL type - foundIsDependency, err := c.buildIsDependency(&collectedIsDependencyLink, nil, true) - if err != nil { + if err := depPkg.setIsDependencyLinks(ctx, inLink.ThisID, c); err != nil { + return nil, err + } + if err := setkv(ctx, isDepCol, inLink, c); err != nil { return nil, err } + outLink = inLink - return foundIsDependency, nil + return c.buildIsDependency(ctx, outLink, nil, true) } // Query IsDependency @@ -162,54 +150,50 @@ func (c *demoClient) IsDependency(ctx context.Context, filter *model.IsDependenc c.m.RLock() defer c.m.RUnlock() funcName := "IsDependency" + if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*isDependencyLink](id, c) + link, err := byIDkv[*isDependencyLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - foundIsDependency, err := c.buildIsDependency(link, filter, true) + foundIsDependency, err := c.buildIsDependency(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.IsDependency{foundIsDependency}, nil } - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Package != nil { - pkgs, err := c.findPackageVersion(filter.Package) + pkgs, err := c.findPackageVersion(ctx, filter.Package) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } foundOne = len(pkgs) > 0 for _, pkg := range pkgs { - search = append(search, pkg.isDependencyLinks...) + search = append(search, pkg.IsDependencyLinks...) } } if !foundOne && filter != nil && filter.DependencyPackage != nil { - if filter.DependencyPackage.Version == nil { - exactPackage, err := c.exactPackageName(filter.DependencyPackage) + if filter.DependencyPackage.Version == nil { // FIXME this logic isn't exactly correct + exactPackage, err := c.exactPackageName(ctx, filter.DependencyPackage) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactPackage != nil { - search = append(search, exactPackage.isDependencyLinks...) + search = append(search, exactPackage.IsDependencyLinks...) foundOne = true } } else { - pkgs, err := c.findPackageVersion(filter.Package) + pkgs, err := c.findPackageVersion(ctx, filter.DependencyPackage) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } foundOne = len(pkgs) > 0 for _, pkg := range pkgs { - search = append(search, pkg.isDependencyLinks...) + search = append(search, pkg.IsDependencyLinks...) } } } @@ -217,19 +201,26 @@ func (c *demoClient) IsDependency(ctx context.Context, filter *model.IsDependenc var out []*model.IsDependency if foundOne { for _, id := range search { - link, err := byID[*isDependencyLink](id, c) + link, err := byIDkv[*isDependencyLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addDepIfMatch(out, filter, link) + out, err = c.addDepIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.isDependencies { - var err error - out, err = c.addDepIfMatch(out, filter, link) + depKeys, err := c.kv.Keys(ctx, isDepCol) + if err != nil { + return nil, err + } + for _, depKey := range depKeys { + link, err := byKeykv[*isDependencyLink](ctx, isDepCol, depKey, c) + if err != nil { + return nil, err + } + out, err = c.addDepIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -239,17 +230,17 @@ func (c *demoClient) IsDependency(ctx context.Context, filter *model.IsDependenc return out, nil } -func (c *demoClient) buildIsDependency(link *isDependencyLink, filter *model.IsDependencySpec, ingestOrIDProvided bool) (*model.IsDependency, error) { +func (c *demoClient) buildIsDependency(ctx context.Context, link *isDependencyLink, filter *model.IsDependencySpec, ingestOrIDProvided bool) (*model.IsDependency, error) { var p *model.Package var dep *model.Package var err error if filter != nil { - p, err = c.buildPackageResponse(link.packageID, filter.Package) + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Package) if err != nil { return nil, err } } else { - p, err = c.buildPackageResponse(link.packageID, nil) + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } @@ -257,12 +248,12 @@ func (c *demoClient) buildIsDependency(link *isDependencyLink, filter *model.IsD if filter != nil && filter.DependencyPackage != nil { depPkgFilter := &model.PkgSpec{Type: filter.DependencyPackage.Type, Namespace: filter.DependencyPackage.Namespace, Name: filter.DependencyPackage.Name} - dep, err = c.buildPackageResponse(link.depPackageID, depPkgFilter) + dep, err = c.buildPackageResponse(ctx, link.DepPackageID, depPkgFilter) if err != nil { return nil, err } } else { - dep, err = c.buildPackageResponse(link.depPackageID, nil) + dep, err = c.buildPackageResponse(ctx, link.DepPackageID, nil) if err != nil { return nil, err } @@ -282,26 +273,26 @@ func (c *demoClient) buildIsDependency(link *isDependencyLink, filter *model.IsD } foundIsDependency := model.IsDependency{ - ID: nodeID(link.id), + ID: link.ThisID, Package: p, DependencyPackage: dep, - VersionRange: link.versionRange, - DependencyType: link.dependencyType, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, + VersionRange: link.VersionRange, + DependencyType: link.DependencyType, + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, } return &foundIsDependency, nil } -func (c *demoClient) addDepIfMatch(out []*model.IsDependency, +func (c *demoClient) addDepIfMatch(ctx context.Context, out []*model.IsDependency, filter *model.IsDependencySpec, link *isDependencyLink) ( []*model.IsDependency, error) { if noMatchIsDep(filter, link) { return out, nil } - foundIsDependency, err := c.buildIsDependency(link, filter, false) + foundIsDependency, err := c.buildIsDependency(ctx, link, filter, false) if err != nil { return nil, err } @@ -313,21 +304,21 @@ func (c *demoClient) addDepIfMatch(out []*model.IsDependency, func noMatchIsDep(filter *model.IsDependencySpec, link *isDependencyLink) bool { if filter != nil { - return noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) || - noMatch(filter.VersionRange, link.versionRange) || - (filter.DependencyType != nil && *filter.DependencyType != link.dependencyType) + return noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) || + noMatch(filter.VersionRange, link.VersionRange) || + (filter.DependencyType != nil && *filter.DependencyType != link.DependencyType) } else { return false } } -func (c *demoClient) matchDependencies(filters []*model.IsDependencySpec, depLinkIDs []uint32) bool { +func (c *demoClient) matchDependencies(ctx context.Context, filters []*model.IsDependencySpec, depLinkIDs []string) bool { var depLinks []*isDependencyLink if len(filters) > 0 { for _, depLinkID := range depLinkIDs { - link, err := byID[*isDependencyLink](depLinkID, c) + link, err := byIDkv[*isDependencyLink](ctx, depLinkID, c) if err != nil { return false } @@ -348,8 +339,8 @@ func (c *demoClient) matchDependencies(filters []*model.IsDependencySpec, depLin match := false for _, depLink := range depLinks { if !noMatchIsDep(filter, depLink) && - (filter.Package == nil || c.matchPackages([]*model.PkgSpec{filter.Package}, []uint32{depLink.packageID})) && - (filter.DependencyPackage == nil || c.matchPackages([]*model.PkgSpec{filter.DependencyPackage}, []uint32{depLink.depPackageID})) { + (filter.Package == nil || c.matchPackages(ctx, []*model.PkgSpec{filter.Package}, []string{depLink.PackageID})) && + (filter.DependencyPackage == nil || c.matchPackages(ctx, []*model.PkgSpec{filter.DependencyPackage}, []string{depLink.DepPackageID})) { match = true break } diff --git a/pkg/assembler/backends/inmem/isDependency_test.go b/pkg/assembler/backends/keyvalue/isDependency_test.go similarity index 94% rename from pkg/assembler/backends/inmem/isDependency_test.go rename to pkg/assembler/backends/keyvalue/isDependency_test.go index 109f842bc8..caead5de40 100644 --- a/pkg/assembler/backends/inmem/isDependency_test.go +++ b/pkg/assembler/backends/keyvalue/isDependency_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -32,6 +33,12 @@ var ( mSpecific = model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion} ) +// func lessID(a, b *model.IsDependency) int { +// ab, _ := json.Marshal(a) +// bb, _ := json.Marshal(b) +// return bytes.Compare(ab, bb) +// } + func TestIsDependency(t *testing.T) { type call struct { P1 *model.PkgInputSpec @@ -444,34 +451,6 @@ func TestIsDependency(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InPkg: []*model.PkgInputSpec{p1, p2, p3}, - Calls: []call{ - { - P1: p1, - P2: p2, - MF: mAll, - ID: &model.IsDependencyInputSpec{}, - }, - { - P1: p2, - P2: p3, - MF: mAll, - ID: &model.IsDependencyInputSpec{}, - }, - { - P1: p1, - P2: p3, - MF: mAll, - ID: &model.IsDependencyInputSpec{}, - }, - }, - Query: &model.IsDependencySpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, { Name: "IsDep from version to version", InPkg: []*model.PkgInputSpec{p2, p3}, @@ -547,12 +526,12 @@ func TestIsDependency(t *testing.T) { ExpID: []*model.IsDependency{ { Package: p3out, - DependencyPackage: p2out, + DependencyPackage: p2outName, Justification: "test justification", }, { Package: p3out, - DependencyPackage: p2outName, + DependencyPackage: p2out, Justification: "test justification", }, }, @@ -564,7 +543,8 @@ func TestIsDependency(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -589,13 +569,8 @@ func TestIsDependency(t *testing.T) { if err != nil { return } - // less := func(a, b *model.Package) bool { return a.Version < b.Version } - // for _, he := range got { - // slices.SortFunc(he.Packages, less) - // } - // for _, he := range test.ExpID { - // slices.SortFunc(he.Packages, less) - // } + //slices.SortFunc(got, lessID) + //slices.SortFunc(test.ExpID, lessID) if diff := cmp.Diff(test.ExpID, got, ignoreID); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) } @@ -654,7 +629,8 @@ func TestIsDependencies(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -744,7 +720,8 @@ func TestIsDependencyNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/isOccurrence.go b/pkg/assembler/backends/keyvalue/isOccurrence.go new file mode 100644 index 0000000000..4a5d031ad7 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/isOccurrence.go @@ -0,0 +1,408 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal isOccurrence + +type isOccurrenceStruct struct { + ThisID string + Pkg string + Source string + Artifact string + Justification string + Origin string + Collector string +} + +func (n *isOccurrenceStruct) ID() string { return n.ThisID } + +func (n *isOccurrenceStruct) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 3) + if n.Pkg != "" && allowedEdges[model.EdgeIsOccurrencePackage] { + out = append(out, n.Pkg) + } + if n.Source != "" && allowedEdges[model.EdgeIsOccurrenceSource] { + out = append(out, n.Source) + } + if allowedEdges[model.EdgeIsOccurrenceArtifact] { + out = append(out, n.Artifact) + } + return out +} + +func (n *isOccurrenceStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convOccurrence(ctx, n) +} + +func (n *isOccurrenceStruct) Key() string { + return strings.Join([]string{ + n.Pkg, + n.Source, + n.Artifact, + n.Justification, + n.Origin, + n.Collector, + }, ":") +} + +// Ingest IngestOccurrences + +func (c *demoClient) IngestOccurrences(ctx context.Context, subjects model.PackageOrSourceInputs, artifacts []*model.ArtifactInputSpec, occurrences []*model.IsOccurrenceInputSpec) ([]*model.IsOccurrence, error) { + var modelIsOccurrences []*model.IsOccurrence + + for i := range occurrences { + var isOccurrence *model.IsOccurrence + var err error + if len(subjects.Packages) > 0 { + subject := model.PackageOrSourceInput{Package: subjects.Packages[i]} + isOccurrence, err = c.IngestOccurrence(ctx, subject, *artifacts[i], *occurrences[i]) + if err != nil { + return nil, gqlerror.Errorf("ingestOccurrence failed with err: %v", err) + } + } else { + subject := model.PackageOrSourceInput{Source: subjects.Sources[i]} + isOccurrence, err = c.IngestOccurrence(ctx, subject, *artifacts[i], *occurrences[i]) + if err != nil { + return nil, gqlerror.Errorf("ingestOccurrence failed with err: %v", err) + } + } + modelIsOccurrences = append(modelIsOccurrences, isOccurrence) + } + return modelIsOccurrences, nil +} + +// Ingest IsOccurrence + +func (c *demoClient) IngestOccurrence(ctx context.Context, subject model.PackageOrSourceInput, artifact model.ArtifactInputSpec, occurrence model.IsOccurrenceInputSpec) (*model.IsOccurrence, error) { + return c.ingestOccurrence(ctx, subject, artifact, occurrence, true) +} + +func (c *demoClient) ingestOccurrence(ctx context.Context, subject model.PackageOrSourceInput, artifact model.ArtifactInputSpec, occurrence model.IsOccurrenceInputSpec, readOnly bool) (*model.IsOccurrence, error) { + funcName := "IngestOccurrence" + + in := &isOccurrenceStruct{ + Justification: occurrence.Justification, + Origin: occurrence.Origin, + Collector: occurrence.Collector, + } + + lock(&c.m, readOnly) + defer unlock(&c.m, readOnly) + + a, err := c.artifactByInput(ctx, &artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: Artifact not found %s", funcName, err) + } + in.Artifact = a.ThisID + + var pkgVer *pkgVersion + if subject.Package != nil { + var err error + pkgVer, err = c.getPackageVerFromInput(ctx, *subject.Package) + if err != nil { + return nil, gqlerror.Errorf("IngestOccurrence :: %v", err) + } + in.Pkg = pkgVer.ThisID + } + + var src *srcNameNode + if subject.Source != nil { + var err error + src, err = c.getSourceNameFromInput(ctx, *subject.Source) + if err != nil { + return nil, gqlerror.Errorf("IngestOccurrence :: %v", err) + } + in.Source = src.ThisID + } + + out, err := byKeykv[*isOccurrenceStruct](ctx, occCol, in.Key(), c) + if err == nil { + return c.convOccurrence(ctx, out) + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + + if readOnly { + c.m.RUnlock() + o, err := c.ingestOccurrence(ctx, subject, artifact, occurrence, false) + c.m.RLock() // relock so that defer unlock does not panic + return o, err + } + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, occCol, in); err != nil { + return nil, err + } + if err := a.setOccurrences(ctx, in.ThisID, c); err != nil { + return nil, err + } + if pkgVer != nil { + if err := pkgVer.setOccurrenceLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + } else { + if err := src.setOccurrenceLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := setkv(ctx, occCol, in, c); err != nil { + return nil, err + } + + return c.convOccurrence(ctx, in) +} + +func (c *demoClient) convOccurrence(ctx context.Context, in *isOccurrenceStruct) (*model.IsOccurrence, error) { + a, err := c.artifactModelByID(ctx, in.Artifact) + if err != nil { + return nil, err + } + o := &model.IsOccurrence{ + ID: in.ThisID, + Artifact: a, + Justification: in.Justification, + Origin: in.Origin, + Collector: in.Collector, + } + if in.Pkg != "" { + p, err := c.buildPackageResponse(ctx, in.Pkg, nil) + if err != nil { + return nil, err + } + o.Subject = p + } else { + s, err := c.buildSourceResponse(ctx, in.Source, nil) + if err != nil { + return nil, err + } + o.Subject = s + } + return o, nil +} + +func (c *demoClient) artifactMatch(ctx context.Context, aID string, artifactSpec *model.ArtifactSpec) bool { + if artifactSpec.Digest == nil && artifactSpec.Algorithm == nil { + return true + } + a, _ := c.artifactExact(ctx, artifactSpec) + if a != nil && a.ID() == aID { + return true + } + m, err := byIDkv[*artStruct](ctx, aID, c) + if err != nil { + return false + } + if artifactSpec.Digest != nil && strings.ToLower(*artifactSpec.Digest) == m.Digest { + return true + } + if artifactSpec.Algorithm != nil && strings.ToLower(*artifactSpec.Algorithm) == m.Algorithm { + return true + } + return false +} + +// Query IsOccurrence + +func (c *demoClient) IsOccurrence(ctx context.Context, filter *model.IsOccurrenceSpec) ([]*model.IsOccurrence, error) { + funcName := "IsOccurrence" + + c.m.RLock() + defer c.m.RUnlock() + + if filter != nil && filter.ID != nil { + link, err := byIDkv[*isOccurrenceStruct](ctx, *filter.ID, c) + if err != nil { + // Not found + return nil, nil + } + // If found by id, ignore rest of fields in spec and return as a match + o, err := c.convOccurrence(ctx, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + return []*model.IsOccurrence{o}, nil + } + + var search []string + foundOne := false + if filter != nil && filter.Artifact != nil { + exactArtifact, err := c.artifactExact(ctx, filter.Artifact) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + if exactArtifact != nil { + search = append(search, exactArtifact.Occurrences...) + foundOne = true + } + } + if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Package != nil { + pkgs, err := c.findPackageVersion(ctx, filter.Subject.Package) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + foundOne = len(pkgs) > 0 + for _, pkg := range pkgs { + search = append(search, pkg.Occurrences...) + } + } + if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { + exactSource, err := c.exactSource(ctx, filter.Subject.Source) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + if exactSource != nil { + search = append(search, exactSource.Occurrences...) + foundOne = true + } + } + + var out []*model.IsOccurrence + if foundOne { + for _, id := range search { + link, err := byIDkv[*isOccurrenceStruct](ctx, id, c) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + out, err = c.addOccIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } else { + occKeys, err := c.kv.Keys(ctx, occCol) + if err != nil { + return nil, err + } + for _, ok := range occKeys { + link, err := byKeykv[*isOccurrenceStruct](ctx, occCol, ok, c) + if err != nil { + return nil, err + } + out, err = c.addOccIfMatch(ctx, out, filter, link) + if err != nil { + return nil, gqlerror.Errorf("%v :: %v", funcName, err) + } + } + } + return out, nil +} + +func (c *demoClient) addOccIfMatch(ctx context.Context, out []*model.IsOccurrence, + filter *model.IsOccurrenceSpec, link *isOccurrenceStruct) ( + []*model.IsOccurrence, error) { + + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) { + return out, nil + } + if filter.Artifact != nil && !c.artifactMatch(ctx, link.Artifact, filter.Artifact) { + return out, nil + } + if filter.Subject != nil { + if filter.Subject.Package != nil { + if link.Pkg == "" { + return out, nil + } + p, err := c.buildPackageResponse(ctx, link.Pkg, filter.Subject.Package) + if err != nil { + return nil, err + } + if p == nil { + return out, nil + } + } else if filter.Subject.Source != nil { + if link.Source == "" { + return out, nil + } + s, err := c.buildSourceResponse(ctx, link.Source, filter.Subject.Source) + if err != nil { + return nil, err + } + if s == nil { + return out, nil + } + } + } + o, err := c.convOccurrence(ctx, link) + if err != nil { + return nil, err + } + return append(out, o), nil +} + +func (c *demoClient) matchOccurrences(ctx context.Context, filters []*model.IsOccurrenceSpec, occLinkIDs []string) bool { + var occLinks []*isOccurrenceStruct + if len(filters) > 0 { + for _, occLinkID := range occLinkIDs { + link, err := byIDkv[*isOccurrenceStruct](ctx, occLinkID, c) + if err != nil { + return false + } + occLinks = append(occLinks, link) + } + + for _, filter := range filters { + if filter == nil { + continue + } + if filter.ID != nil { + // Check by ID if present + if !c.isIDPresent(*filter.ID, occLinkIDs) { + return false + } + } else { + // Otherwise match spec information + match := false + for _, link := range occLinks { + if !noMatch(filter.Justification, link.Justification) && + !noMatch(filter.Origin, link.Origin) && + !noMatch(filter.Collector, link.Collector) && + c.matchArtifacts(ctx, []*model.ArtifactSpec{filter.Artifact}, []string{link.Artifact}) { + + if filter.Subject != nil { + if filter.Subject.Package != nil && !c.matchPackages(ctx, []*model.PkgSpec{filter.Subject.Package}, []string{link.Pkg}) { + continue + } else if filter.Subject.Source != nil { + src, err := c.exactSource(ctx, filter.Subject.Source) + if err != nil || src == nil { + continue + } + } + } + match = true + break + } + } + if !match { + return false + } + } + } + } + return true +} diff --git a/pkg/assembler/backends/inmem/isOccurrence_test.go b/pkg/assembler/backends/keyvalue/isOccurrence_test.go similarity index 96% rename from pkg/assembler/backends/inmem/isOccurrence_test.go rename to pkg/assembler/backends/keyvalue/isOccurrence_test.go index b5400b62db..2cddf7f3ff 100644 --- a/pkg/assembler/backends/inmem/isOccurrence_test.go +++ b/pkg/assembler/backends/keyvalue/isOccurrence_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" jsoniter "github.com/json-iterator/go" @@ -208,9 +209,7 @@ var s1out = &model.Source{ Namespaces: []*model.SourceNamespace{{ Namespace: "github.com/jeff", Names: []*model.SourceName{{ - Name: "myrepo", - Tag: ptrfrom.String(""), - Commit: ptrfrom.String(""), + Name: "myrepo", }}, }}, } @@ -226,9 +225,7 @@ var s2out = &model.Source{ Namespaces: []*model.SourceNamespace{{ Namespace: "github.com/bob", Names: []*model.SourceName{{ - Name: "bobsrepo", - Tag: ptrfrom.String(""), - Commit: ptrfrom.String(""), + Name: "bobsrepo", }}, }}, } @@ -535,26 +532,6 @@ func TestOccurrence(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InPkg: []*model.PkgInputSpec{p1}, - InArt: []*model.ArtifactInputSpec{a1}, - Calls: []call{ - { - PkgSrc: model.PackageOrSourceInput{ - Package: p1, - }, - Artifact: a1, - Occurrence: &model.IsOccurrenceInputSpec{ - Justification: "justification", - }, - }, - }, - Query: &model.IsOccurrenceSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -562,7 +539,8 @@ func TestOccurrence(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -676,7 +654,8 @@ func TestIngestOccurrences(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -785,7 +764,8 @@ func TestOccurrenceNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/license.go b/pkg/assembler/backends/keyvalue/license.go new file mode 100644 index 0000000000..79a8c032a5 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/license.go @@ -0,0 +1,203 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal data: Licenses +type licStruct struct { + ThisID string + Name string + Inline string + ListVersion string + CertifyLegals []string +} + +func (n *licStruct) ID() string { return n.ThisID } +func (n *licStruct) Key() string { + return strings.Join([]string{ + n.Name, + n.ListVersion, + }, ":") +} + +func (n *licStruct) Neighbors(allowedEdges edgeMap) []string { + if allowedEdges[model.EdgeLicenseCertifyLegal] { + return n.CertifyLegals + } + return nil +} + +func (n *licStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convLicense(n), nil +} + +func (n *licStruct) setCertifyLegals(ctx context.Context, id string, c *demoClient) error { + n.CertifyLegals = append(n.CertifyLegals, id) + return setkv(ctx, licenseCol, n, c) +} + +func (c *demoClient) licenseByInput(ctx context.Context, b *model.LicenseInputSpec) (*licStruct, error) { + in := &licStruct{ + Name: b.Name, + ListVersion: nilToEmpty(b.ListVersion), + } + return byKeykv[*licStruct](ctx, licenseCol, in.Key(), c) +} + +// Ingest Licenses + +func (c *demoClient) IngestLicenses(ctx context.Context, licenses []*model.LicenseInputSpec) ([]*model.License, error) { + var modelLicenses []*model.License + for _, lic := range licenses { + modelLic, err := c.IngestLicense(ctx, lic) + if err != nil { + return nil, gqlerror.Errorf("ingestLicense failed with err: %v", err) + } + modelLicenses = append(modelLicenses, modelLic) + } + return modelLicenses, nil +} + +func (c *demoClient) IngestLicense(ctx context.Context, license *model.LicenseInputSpec) (*model.License, error) { + return c.ingestLicense(ctx, license, true) +} + +func (c *demoClient) ingestLicense(ctx context.Context, license *model.LicenseInputSpec, readOnly bool) (*model.License, error) { + in := &licStruct{ + Name: license.Name, + Inline: nilToEmpty(license.Inline), + ListVersion: nilToEmpty(license.ListVersion), + } + + lock(&c.m, readOnly) + defer unlock(&c.m, readOnly) + + out, err := byKeykv[*licStruct](ctx, licenseCol, in.Key(), c) + if err == nil { + return c.convLicense(out), nil + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + if readOnly { + c.m.RUnlock() + a, err := c.ingestLicense(ctx, license, false) + c.m.RLock() // relock so that defer unlock does not panic + return a, err + } + in.ThisID = c.getNextID() + + if err := c.addToIndex(ctx, licenseCol, in); err != nil { + return nil, err + } + if err := setkv(ctx, licenseCol, in, c); err != nil { + return nil, err + } + + return c.convLicense(in), nil +} + +func (c *demoClient) licenseExact(ctx context.Context, licenseSpec *model.LicenseSpec) (*licStruct, error) { + if licenseSpec == nil { + return nil, nil + } + + // If ID is provided, try to look up + if licenseSpec.ID != nil { + a, err := byIDkv[*licStruct](ctx, *licenseSpec.ID, c) + if err == nil { + // If found by id, ignore rest of fields in spec and return as a match + return a, nil + } + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + // Not found + return nil, nil + } + + if licenseSpec.Name != nil { + in := &licStruct{ + Name: *licenseSpec.Name, + ListVersion: nilToEmpty(licenseSpec.ListVersion), + } + out, err := byKeykv[*licStruct](ctx, licenseCol, in.Key(), c) + if err == nil { + return out, nil + } + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + } + return nil, nil +} + +// Query Licenses + +func (c *demoClient) Licenses(ctx context.Context, licenseSpec *model.LicenseSpec) ([]*model.License, error) { + c.m.RLock() + defer c.m.RUnlock() + a, err := c.licenseExact(ctx, licenseSpec) + if err != nil { + return nil, gqlerror.Errorf("Licenses :: invalid spec %s", err) + } + if a != nil { + return []*model.License{c.convLicense(a)}, nil + } + + var rv []*model.License + lKeys, err := c.kv.Keys(ctx, licenseCol) + if err != nil { + return nil, err + } + for _, lk := range lKeys { + l, err := byKeykv[*licStruct](ctx, licenseCol, lk, c) + if err != nil { + return nil, err + } + if noMatch(licenseSpec.Name, l.Name) || + noMatch(licenseSpec.ListVersion, l.ListVersion) || + noMatch(licenseSpec.Inline, l.Inline) { + continue + } + rv = append(rv, c.convLicense(l)) + } + return rv, nil +} + +func (c *demoClient) convLicense(a *licStruct) *model.License { + rv := &model.License{ + ID: a.ThisID, + Name: a.Name, + } + if a.Inline != "" { + rv.Inline = &a.Inline + } + if a.ListVersion != "" { + rv.ListVersion = &a.ListVersion + } + return rv +} diff --git a/pkg/assembler/backends/inmem/license_test.go b/pkg/assembler/backends/keyvalue/license_test.go similarity index 95% rename from pkg/assembler/backends/inmem/license_test.go rename to pkg/assembler/backends/keyvalue/license_test.go index 44824ab143..b84b09644f 100644 --- a/pkg/assembler/backends/inmem/license_test.go +++ b/pkg/assembler/backends/keyvalue/license_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -142,14 +143,6 @@ func TestLicense(t *testing.T) { }, Exp: nil, }, - { - Name: "Query invalid ID", - Ingests: []*model.LicenseInputSpec{l1, l2, l3}, - Query: &model.LicenseSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -157,7 +150,8 @@ func TestLicense(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -207,7 +201,8 @@ func TestIngestLicenses(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/path.go b/pkg/assembler/backends/keyvalue/path.go similarity index 63% rename from pkg/assembler/backends/inmem/path.go rename to pkg/assembler/backends/keyvalue/path.go index 922d982db1..fd71b11f94 100644 --- a/pkg/assembler/backends/inmem/path.go +++ b/pkg/assembler/backends/keyvalue/path.go @@ -13,11 +13,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "fmt" + "strings" "github.com/vektah/gqlparser/v2/gqlerror" @@ -39,28 +40,14 @@ func processUsingOnly(usingOnly []model.Edge) edgeMap { } func (c *demoClient) Path(ctx context.Context, source string, target string, maxPathLength int, usingOnly []model.Edge) ([]model.Node, error) { - sourceID, err := strconv.ParseUint(source, 10, 32) - if err != nil { - return nil, err - } - targetID, err := strconv.ParseUint(target, 10, 32) - if err != nil { - return nil, err - } - c.m.RLock() defer c.m.RUnlock() - return c.bfs(uint32(sourceID), uint32(targetID), maxPathLength, processUsingOnly(usingOnly)) + return c.bfs(ctx, source, target, maxPathLength, processUsingOnly(usingOnly)) } func (c *demoClient) Neighbors(ctx context.Context, source string, usingOnly []model.Edge) ([]model.Node, error) { - id, err := strconv.ParseUint(source, 10, 32) - if err != nil { - return nil, err - } - c.m.RLock() - neighbors, err := c.neighborsFromId(uint32(id), processUsingOnly(usingOnly)) + neighbors, err := c.neighborsFromId(ctx, source, processUsingOnly(usingOnly)) if err != nil { c.m.RUnlock() return nil, err @@ -69,44 +56,36 @@ func (c *demoClient) Neighbors(ctx context.Context, source string, usingOnly []m c.m.RLock() defer c.m.RUnlock() - return c.buildModelNodes(neighbors) + return c.Nodes(ctx, neighbors) } -func (c *demoClient) buildModelNodes(nodeIDs []uint32) ([]model.Node, error) { - out := make([]model.Node, len(nodeIDs)) - - for i, nodeID := range nodeIDs { - node, ok := c.index[nodeID] - if !ok { - return nil, gqlerror.Errorf("Internal data error: got invalid node id %d", nodeID) - } - var err error - - out[i], err = node.BuildModelNode(c) - if err != nil { - return nil, err - } +func (c *demoClient) neighborsFromId(ctx context.Context, id string, allowedEdges edgeMap) ([]string, error) { + var k string + if err := c.kv.Get(ctx, indexCol, id, &k); err != nil { + return nil, fmt.Errorf("%w : id not found in index %q", err, id) } - return out, nil -} + sub := strings.SplitN(k, ":", 2) + if len(sub) != 2 { + return nil, fmt.Errorf("Bad value was stored in index map: %v", k) + } -func (c *demoClient) neighborsFromId(id uint32, allowedEdges edgeMap) ([]uint32, error) { - node, ok := c.index[id] - if !ok { - return nil, gqlerror.Errorf("ID does not match existing node") + node := typeColMap(sub[0]) + if err := c.kv.Get(ctx, sub[0], sub[1], &node); err != nil { + return nil, err } + return node.Neighbors(allowedEdges), nil } -func (c *demoClient) bfs(from, to uint32, maxLength int, allowedEdges edgeMap) ([]model.Node, error) { - queue := make([]uint32, 0) // the queue of nodes in bfs +func (c *demoClient) bfs(ctx context.Context, from, to string, maxLength int, allowedEdges edgeMap) ([]model.Node, error) { + queue := make([]string, 0) // the queue of nodes in bfs type dfsNode struct { expanded bool // true once all node neighbors are added to queue - parent uint32 + parent string depth int } - nodeMap := map[uint32]dfsNode{} + nodeMap := map[string]dfsNode{} nodeMap[from] = dfsNode{} queue = append(queue, from) @@ -126,7 +105,7 @@ func (c *demoClient) bfs(from, to uint32, maxLength int, allowedEdges edgeMap) ( break } - neighbors, err := c.neighborsFromId(now, allowedEdges) + neighbors, err := c.neighborsFromId(ctx, now, allowedEdges) if err != nil { return nil, err } @@ -153,7 +132,7 @@ func (c *demoClient) bfs(from, to uint32, maxLength int, allowedEdges edgeMap) ( return nil, gqlerror.Errorf("No path found up to specified length") } - reversedPath := []uint32{} + reversedPath := []string{} now := to for now != from { reversedPath = append(reversedPath, now) @@ -162,28 +141,34 @@ func (c *demoClient) bfs(from, to uint32, maxLength int, allowedEdges edgeMap) ( reversedPath = append(reversedPath, now) // reverse path - path := make([]uint32, len(reversedPath)) + path := make([]string, len(reversedPath)) for i, x := range reversedPath { path[len(reversedPath)-i-1] = x } - return c.buildModelNodes(path) + return c.Nodes(ctx, path) } -func (c *demoClient) Node(ctx context.Context, source string) (model.Node, error) { - id, err := strconv.ParseUint(source, 10, 32) - if err != nil { - return nil, err - } - +func (c *demoClient) Node(ctx context.Context, id string) (model.Node, error) { c.m.RLock() defer c.m.RUnlock() - node, ok := c.index[uint32(id)] - if !ok { - return nil, gqlerror.Errorf("Node: got invalid node id %d", id) + + var k string + if err := c.kv.Get(ctx, indexCol, id, &k); err != nil { + return nil, fmt.Errorf("%w : id not found in index %q", err, id) + } + + sub := strings.SplitN(k, ":", 2) + if len(sub) != 2 { + return nil, fmt.Errorf("Bad value was stored in index map: %v", k) + } + + node := typeColMap(sub[0]) + if err := c.kv.Get(ctx, sub[0], sub[1], &node); err != nil { + return nil, err } - out, err := node.BuildModelNode(c) + out, err := node.BuildModelNode(ctx, c) if err != nil { return nil, gqlerror.Errorf("Node: could not build node: %v", err) } diff --git a/pkg/assembler/backends/inmem/path_test.go b/pkg/assembler/backends/keyvalue/path_test.go similarity index 99% rename from pkg/assembler/backends/inmem/path_test.go rename to pkg/assembler/backends/keyvalue/path_test.go index 4927c6396b..b4dabc5fae 100644 --- a/pkg/assembler/backends/inmem/path_test.go +++ b/pkg/assembler/backends/keyvalue/path_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -22,6 +22,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/internal/testing/testdata" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" @@ -176,7 +177,7 @@ func Test_Nodes(t *testing.T) { }, { name: "source", srcInput: testdata.S1, - want: []model.Node{testdata.S1out}, + want: []model.Node{s1out}, wantErr: false, }, { name: "vulnerability", @@ -253,7 +254,7 @@ func Test_Nodes(t *testing.T) { }, }, want: []model.Node{&model.CertifyScorecard{ - Source: testdata.S2out, + Source: s2out, Scorecard: &model.Scorecard{ Checks: []*model.ScorecardCheck{}, Origin: "test origin", @@ -409,7 +410,7 @@ func Test_Nodes(t *testing.T) { }, want: []model.Node{&model.HasSourceAt{ Package: testdata.P2out, - Source: testdata.S1out, + Source: s1out, }}, }, { name: "isDependency", @@ -525,7 +526,8 @@ func Test_Nodes(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/pkg.go b/pkg/assembler/backends/keyvalue/pkg.go new file mode 100644 index 0000000000..cde4e30280 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/pkg.go @@ -0,0 +1,1007 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal data: Packages +type pkgType struct { + ThisID string + Type string + Namespaces []string +} +type pkgNamespace struct { + ThisID string + Parent string + Namespace string + Names []string +} +type pkgName struct { + ThisID string + Parent string + Name string + Versions []string + SrcMapLinks []string + IsDependencyLinks []string + BadLinks []string + GoodLinks []string + HasMetadataLinks []string + PointOfContactLinks []string +} +type pkgVersion struct { + ThisID string + Parent string + Version string + Subpath string + Qualifiers map[string]string + SrcMapLinks []string + IsDependencyLinks []string + Occurrences []string + CertifyVulnLinks []string + HasSBOMs []string + VexLinks []string + BadLinks []string + GoodLinks []string + HasMetadataLinks []string + PointOfContactLinks []string + PkgEquals []string + CertifyLegals []string +} + +// Be type safe, don't use any / interface{} +type pkgNameOrVersion interface { + setSrcMapLinks(ctx context.Context, ID string, c *demoClient) error + getSrcMapLinks() []string + setIsDependencyLinks(ctx context.Context, ID string, c *demoClient) error + getIsDependencyLinks() []string + setCertifyBadLinks(ctx context.Context, ID string, c *demoClient) error + getCertifyBadLinks() []string + setCertifyGoodLinks(ctx context.Context, ID string, c *demoClient) error + getCertifyGoodLinks() []string + setHasMetadataLinks(ctx context.Context, ID string, c *demoClient) error + getHasMetadataLinks() []string + setPointOfContactLinks(ctx context.Context, ID string, c *demoClient) error + getPointOfContactLinks() []string + + node +} + +var _ pkgNameOrVersion = &pkgName{} +var _ pkgNameOrVersion = &pkgVersion{} + +func (n *pkgType) ID() string { return n.ThisID } +func (n *pkgNamespace) ID() string { return n.ThisID } +func (n *pkgName) ID() string { return n.ThisID } +func (n *pkgVersion) ID() string { return n.ThisID } + +func (n *pkgType) Neighbors(allowedEdges edgeMap) []string { + if allowedEdges[model.EdgePackageTypePackageNamespace] { + return n.Namespaces + } + return nil +} +func (n *pkgNamespace) Neighbors(allowedEdges edgeMap) []string { + var out []string + if allowedEdges[model.EdgePackageNamespacePackageName] { + out = append(out, n.Names...) + } + if allowedEdges[model.EdgePackageNamespacePackageType] { + out = append(out, n.Parent) + } + return out +} +func (n *pkgName) Neighbors(allowedEdges edgeMap) []string { + var out []string + if allowedEdges[model.EdgePackageNamePackageNamespace] { + out = append(out, n.Parent) + } + if allowedEdges[model.EdgePackageNamePackageVersion] { + out = append(out, n.Versions...) + } + if allowedEdges[model.EdgePackageHasSourceAt] { + out = append(out, n.SrcMapLinks...) + } + if allowedEdges[model.EdgePackageIsDependency] { + out = append(out, n.IsDependencyLinks...) + } + if allowedEdges[model.EdgePackageCertifyBad] { + out = append(out, n.BadLinks...) + } + if allowedEdges[model.EdgePackageCertifyGood] { + out = append(out, n.GoodLinks...) + } + if allowedEdges[model.EdgePackageHasMetadata] { + out = append(out, n.HasMetadataLinks...) + } + if allowedEdges[model.EdgePackagePointOfContact] { + out = append(out, n.PointOfContactLinks...) + } + + return out +} +func (n *pkgVersion) Neighbors(allowedEdges edgeMap) []string { + var out []string + if allowedEdges[model.EdgePackageVersionPackageName] { + out = append(out, n.Parent) + } + if allowedEdges[model.EdgePackageHasSourceAt] { + out = append(out, n.SrcMapLinks...) + } + if allowedEdges[model.EdgePackageIsDependency] { + out = append(out, n.IsDependencyLinks...) + } + if allowedEdges[model.EdgePackageIsOccurrence] { + out = append(out, n.Occurrences...) + } + if allowedEdges[model.EdgePackageCertifyVuln] { + out = append(out, n.CertifyVulnLinks...) + } + if allowedEdges[model.EdgePackageHasSbom] { + out = append(out, n.HasSBOMs...) + } + if allowedEdges[model.EdgePackageCertifyVexStatement] { + out = append(out, n.VexLinks...) + } + if allowedEdges[model.EdgePackageCertifyBad] { + out = append(out, n.BadLinks...) + } + if allowedEdges[model.EdgePackageCertifyGood] { + out = append(out, n.GoodLinks...) + } + if allowedEdges[model.EdgePackagePkgEqual] { + out = append(out, n.PkgEquals...) + } + if allowedEdges[model.EdgePackageHasMetadata] { + out = append(out, n.HasMetadataLinks...) + } + if allowedEdges[model.EdgePackagePointOfContact] { + out = append(out, n.PointOfContactLinks...) + } + if allowedEdges[model.EdgePackageCertifyLegal] { + out = append(out, n.CertifyLegals...) + } + + return out +} + +func (n *pkgType) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildPackageResponse(ctx, n.ThisID, nil) +} +func (n *pkgNamespace) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildPackageResponse(ctx, n.ThisID, nil) +} +func (n *pkgName) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildPackageResponse(ctx, n.ThisID, nil) +} +func (n *pkgVersion) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildPackageResponse(ctx, n.ThisID, nil) +} + +// hasSourceAt back edges +func (p *pkgName) setSrcMapLinks(ctx context.Context, id string, c *demoClient) error { + p.SrcMapLinks = append(p.SrcMapLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setSrcMapLinks(ctx context.Context, id string, c *demoClient) error { + p.SrcMapLinks = append(p.SrcMapLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getSrcMapLinks() []string { return p.SrcMapLinks } +func (p *pkgVersion) getSrcMapLinks() []string { return p.SrcMapLinks } + +// isDependency back edges +func (p *pkgName) setIsDependencyLinks(ctx context.Context, id string, c *demoClient) error { + p.IsDependencyLinks = append(p.IsDependencyLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setIsDependencyLinks(ctx context.Context, id string, c *demoClient) error { + p.IsDependencyLinks = append(p.IsDependencyLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getIsDependencyLinks() []string { return p.IsDependencyLinks } +func (p *pkgVersion) getIsDependencyLinks() []string { return p.IsDependencyLinks } + +// isOccurrence back edges +func (p *pkgVersion) setOccurrenceLinks(ctx context.Context, id string, c *demoClient) error { + p.Occurrences = append(p.Occurrences, id) + return setkv(ctx, pkgVerCol, p, c) +} + +// certifyVulnerability back edges +func (p *pkgVersion) setVulnerabilityLinks(ctx context.Context, id string, c *demoClient) error { + p.CertifyVulnLinks = append(p.CertifyVulnLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} + +// certifyVexStatement back edges +func (p *pkgVersion) setVexLinks(ctx context.Context, id string, c *demoClient) error { + p.VexLinks = append(p.VexLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} + +// hasSBOM back edges +func (p *pkgVersion) setHasSBOM(ctx context.Context, id string, c *demoClient) error { + p.HasSBOMs = append(p.HasSBOMs, id) + return setkv(ctx, pkgVerCol, p, c) +} + +// certifyBad back edges +func (p *pkgName) setCertifyBadLinks(ctx context.Context, id string, c *demoClient) error { + p.BadLinks = append(p.BadLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setCertifyBadLinks(ctx context.Context, id string, c *demoClient) error { + p.BadLinks = append(p.BadLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getCertifyBadLinks() []string { return p.BadLinks } +func (p *pkgVersion) getCertifyBadLinks() []string { return p.BadLinks } + +// certifyGood back edges +func (p *pkgName) setCertifyGoodLinks(ctx context.Context, id string, c *demoClient) error { + p.GoodLinks = append(p.GoodLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setCertifyGoodLinks(ctx context.Context, id string, c *demoClient) error { + p.GoodLinks = append(p.GoodLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getCertifyGoodLinks() []string { return p.GoodLinks } +func (p *pkgVersion) getCertifyGoodLinks() []string { return p.GoodLinks } + +// hasMetadata back edges +func (p *pkgName) setHasMetadataLinks(ctx context.Context, id string, c *demoClient) error { + p.HasMetadataLinks = append(p.HasMetadataLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setHasMetadataLinks(ctx context.Context, id string, c *demoClient) error { + p.HasMetadataLinks = append(p.HasMetadataLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getHasMetadataLinks() []string { return p.HasMetadataLinks } +func (p *pkgVersion) getHasMetadataLinks() []string { return p.HasMetadataLinks } + +// pointOfContact back edges +func (p *pkgName) setPointOfContactLinks(ctx context.Context, id string, c *demoClient) error { + p.PointOfContactLinks = append(p.PointOfContactLinks, id) + return setkv(ctx, pkgNameCol, p, c) +} +func (p *pkgVersion) setPointOfContactLinks(ctx context.Context, id string, c *demoClient) error { + p.PointOfContactLinks = append(p.PointOfContactLinks, id) + return setkv(ctx, pkgVerCol, p, c) +} +func (p *pkgName) getPointOfContactLinks() []string { return p.PointOfContactLinks } +func (p *pkgVersion) getPointOfContactLinks() []string { return p.PointOfContactLinks } + +// pkgEqual back edges +func (p *pkgVersion) setPkgEquals(ctx context.Context, id string, c *demoClient) error { + p.PkgEquals = append(p.PkgEquals, id) + return setkv(ctx, pkgVerCol, p, c) +} + +func (p *pkgVersion) setCertifyLegals(ctx context.Context, id string, c *demoClient) error { + p.CertifyLegals = append(p.CertifyLegals, id) + return setkv(ctx, pkgVerCol, p, c) +} + +func (n *pkgType) Key() string { + return n.Type +} + +func (n *pkgType) addNamespace(ctx context.Context, ns string, c *demoClient) error { + n.Namespaces = append(n.Namespaces, ns) + return setkv(ctx, pkgTypeCol, n, c) +} + +func (n *pkgNamespace) Key() string { + return strings.Join([]string{ + n.Parent, + n.Namespace, + }, ":") +} + +func (n *pkgNamespace) addName(ctx context.Context, name string, c *demoClient) error { + n.Names = append(n.Names, name) + return setkv(ctx, pkgNSCol, n, c) +} + +func (n *pkgName) Key() string { + return strings.Join([]string{ + n.Parent, + n.Name, + }, ":") +} + +func (n *pkgName) addVersion(ctx context.Context, ver string, c *demoClient) error { + n.Versions = append(n.Versions, ver) + return setkv(ctx, pkgNameCol, n, c) +} + +func (n *pkgVersion) Key() string { + return strings.Join([]string{ + n.Parent, + hashVersionHelper(n.Version, n.Subpath, n.Qualifiers), + }, ":") +} + +// Ingest Package + +func (c *demoClient) IngestPackages(ctx context.Context, pkgs []*model.PkgInputSpec) ([]*model.Package, error) { + var modelPkgs []*model.Package + for _, pkg := range pkgs { + modelPkg, err := c.IngestPackage(ctx, *pkg) + if err != nil { + return nil, gqlerror.Errorf("ingestPackage failed with err: %v", err) + } + modelPkgs = append(modelPkgs, modelPkg) + } + return modelPkgs, nil +} + +func (c *demoClient) IngestPackage(ctx context.Context, input model.PkgInputSpec) (*model.Package, error) { + inType := &pkgType{ + Type: input.Type, + } + c.m.RLock() + outType, err := byKeykv[*pkgType](ctx, pkgTypeCol, inType.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outType, err = byKeykv[*pkgType](ctx, pkgTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inType.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pkgTypeCol, inType); err != nil { + return nil, err + } + if err := setkv(ctx, pkgTypeCol, inType, c); err != nil { + return nil, err + } + outType = inType + } + c.m.Unlock() + } + + inNamespace := &pkgNamespace{ + Parent: outType.ThisID, + Namespace: nilToEmpty(input.Namespace), + } + c.m.RLock() + outNamespace, err := byKeykv[*pkgNamespace](ctx, pkgNSCol, inNamespace.Key(), c) + c.m.RUnlock() + if err != nil { + c.m.Lock() + outNamespace, err = byKeykv[*pkgNamespace](ctx, pkgNSCol, inNamespace.Key(), c) + if err != nil { + inNamespace.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pkgNSCol, inNamespace); err != nil { + return nil, err + } + if err := setkv(ctx, pkgNSCol, inNamespace, c); err != nil { + return nil, err + } + if err := outType.addNamespace(ctx, inNamespace.ThisID, c); err != nil { + return nil, err + } + outNamespace = inNamespace + } + c.m.Unlock() + } + + inName := &pkgName{ + Parent: outNamespace.ThisID, + Name: input.Name, + } + c.m.RLock() + outName, err := byKeykv[*pkgName](ctx, pkgNameCol, inName.Key(), c) + c.m.RUnlock() + if err != nil { + c.m.Lock() + outName, err = byKeykv[*pkgName](ctx, pkgNameCol, inName.Key(), c) + if err != nil { + inName.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pkgNameCol, inName); err != nil { + return nil, err + } + if err := setkv(ctx, pkgNameCol, inName, c); err != nil { + return nil, err + } + if err := outNamespace.addName(ctx, inName.ThisID, c); err != nil { + return nil, err + } + outName = inName + } + c.m.Unlock() + } + + inVersion := &pkgVersion{ + Parent: outName.ThisID, + Version: nilToEmpty(input.Version), + Subpath: nilToEmpty(input.Subpath), + Qualifiers: getQualifiersFromInput(input.Qualifiers), + } + c.m.RLock() + outVersion, err := byKeykv[*pkgVersion](ctx, pkgVerCol, inVersion.Key(), c) + c.m.RUnlock() + if err != nil { + c.m.Lock() + outVersion, err = byKeykv[*pkgVersion](ctx, pkgVerCol, inVersion.Key(), c) + if err != nil { + inVersion.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pkgVerCol, inVersion); err != nil { + return nil, err + } + if err := setkv(ctx, pkgVerCol, inVersion, c); err != nil { + return nil, err + } + if err := outName.addVersion(ctx, inVersion.ThisID, c); err != nil { + return nil, err + } + outVersion = inVersion + } + c.m.Unlock() + } + + // build return GraphQL type + c.m.RLock() + defer c.m.RUnlock() + return c.buildPackageResponse(ctx, outVersion.ThisID, nil) +} + +func hashVersionHelper(version string, subpath string, qualifiers map[string]string) string { + // first sort the qualifiers + qualifierSlice := make([]string, 0, len(qualifiers)) + for key, value := range qualifiers { + qualifierSlice = append(qualifierSlice, fmt.Sprintf("%s:%s", key, value)) + } + slices.Sort(qualifierSlice) + qualifiersStr := strings.Join(qualifierSlice, ",") + + canonicalVersion := fmt.Sprintf("%s,%s,%s", version, subpath, qualifiersStr) + return canonicalVersion + // digest := sha256.Sum256([]byte(canonicalVersion)) + // return fmt.Sprintf("%x", digest) +} + +// Query Package +func (c *demoClient) Packages(ctx context.Context, filter *model.PkgSpec) ([]*model.Package, error) { + c.m.RLock() + defer c.m.RUnlock() + if filter != nil && filter.ID != nil { + p, err := c.buildPackageResponse(ctx, *filter.ID, filter) + if err != nil { + if errors.Is(err, errNotFound) { + // not found + return nil, nil + } + return nil, err + } + return []*model.Package{p}, nil + } + + out := []*model.Package{} + if filter != nil && filter.Type != nil { + inType := &pkgType{ + Type: *filter.Type, + } + pkgTypeNode, err := byKeykv[*pkgType](ctx, pkgTypeCol, inType.Key(), c) + if err == nil { + pNamespaces := c.buildPkgNamespace(ctx, pkgTypeNode, filter) + if len(pNamespaces) > 0 { + out = append(out, &model.Package{ + ID: pkgTypeNode.ThisID, + Type: pkgTypeNode.Type, + Namespaces: pNamespaces, + }) + } + } + } else { + typeKeys, err := c.kv.Keys(ctx, pkgTypeCol) + if err != nil { + return nil, err + } + for _, tk := range typeKeys { + pkgTypeNode, err := byKeykv[*pkgType](ctx, pkgTypeCol, tk, c) + if err != nil { + return nil, err + } + pNamespaces := c.buildPkgNamespace(ctx, pkgTypeNode, filter) + if len(pNamespaces) > 0 { + out = append(out, &model.Package{ + ID: pkgTypeNode.ThisID, + Type: pkgTypeNode.Type, + Namespaces: pNamespaces, + }) + } + } + } + return out, nil +} + +func (c *demoClient) buildPkgNamespace(ctx context.Context, pkgTypeNode *pkgType, filter *model.PkgSpec) []*model.PackageNamespace { + pNamespaces := []*model.PackageNamespace{} + if filter != nil && filter.Namespace != nil { + inNS := &pkgNamespace{ + Parent: pkgTypeNode.ThisID, + Namespace: *filter.Namespace, + } + pkgNS, err := byKeykv[*pkgNamespace](ctx, pkgNSCol, inNS.Key(), c) + if err == nil { + pns := c.buildPkgName(ctx, pkgNS, filter) + if len(pns) > 0 { + pNamespaces = append(pNamespaces, &model.PackageNamespace{ + ID: pkgNS.ThisID, + Namespace: pkgNS.Namespace, + Names: pns, + }) + } + } + } else { + for _, nsID := range pkgTypeNode.Namespaces { + pkgNS, err := byIDkv[*pkgNamespace](ctx, nsID, c) + if err != nil { + continue + } + pns := c.buildPkgName(ctx, pkgNS, filter) + if len(pns) > 0 { + pNamespaces = append(pNamespaces, &model.PackageNamespace{ + ID: pkgNS.ThisID, + Namespace: pkgNS.Namespace, + Names: pns, + }) + } + } + } + return pNamespaces +} + +func (c *demoClient) buildPkgName(ctx context.Context, pkgNS *pkgNamespace, filter *model.PkgSpec) []*model.PackageName { + pns := []*model.PackageName{} + if filter != nil && filter.Name != nil { + inName := &pkgName{ + Parent: pkgNS.ThisID, + Name: *filter.Name, + } + pkgNameNode, err := byKeykv[*pkgName](ctx, pkgNameCol, inName.Key(), c) + if err == nil { + pvs := c.buildPkgVersion(ctx, pkgNameNode, filter) + if len(pvs) > 0 { + pns = append(pns, &model.PackageName{ + ID: pkgNameNode.ThisID, + Name: pkgNameNode.Name, + Versions: pvs, + }) + } + } + } else { + for _, nameID := range pkgNS.Names { + pkgNameNode, err := byIDkv[*pkgName](ctx, nameID, c) + if err != nil { + continue + } + pvs := c.buildPkgVersion(ctx, pkgNameNode, filter) + if len(pvs) > 0 { + pns = append(pns, &model.PackageName{ + ID: pkgNameNode.ThisID, + Name: pkgNameNode.Name, + Versions: pvs, + }) + } + } + } + return pns +} + +func (c *demoClient) buildPkgVersion(ctx context.Context, pkgNameNode *pkgName, filter *model.PkgSpec) []*model.PackageVersion { + pvs := []*model.PackageVersion{} + if filter != nil && + filter.Version != nil && + filter.Subpath != nil && + ((len(filter.Qualifiers) > 0) || + (filter.MatchOnlyEmptyQualifiers != nil && *filter.MatchOnlyEmptyQualifiers)) { + inVer := &pkgVersion{ + Parent: pkgNameNode.ThisID, + Version: *filter.Version, + Subpath: *filter.Subpath, + } + if filter.MatchOnlyEmptyQualifiers == nil || !*filter.MatchOnlyEmptyQualifiers { + inVer.Qualifiers = getQualifiersFromFilter(filter.Qualifiers) + } + pkgVer, err := byKeykv[*pkgVersion](ctx, pkgVerCol, inVer.Key(), c) + if err == nil { + pvs = append(pvs, &model.PackageVersion{ + ID: pkgVer.ThisID, + Version: pkgVer.Version, + Subpath: pkgVer.Subpath, + Qualifiers: getCollectedPackageQualifiers(pkgVer.Qualifiers), + }) + } + return pvs + } + + for _, verID := range pkgNameNode.Versions { + pkgVer, err := byIDkv[*pkgVersion](ctx, verID, c) + if err != nil { + continue + } + if filter != nil && noMatch(filter.Version, pkgVer.Version) { + continue + } + if filter != nil && noMatch(filter.Subpath, pkgVer.Subpath) { + continue + } + if filter != nil && noMatchQualifiers(filter, pkgVer.Qualifiers) { + continue + } + pvs = append(pvs, &model.PackageVersion{ + ID: pkgVer.ThisID, + Version: pkgVer.Version, + Subpath: pkgVer.Subpath, + Qualifiers: getCollectedPackageQualifiers(pkgVer.Qualifiers), + }) + } + return pvs +} + +// Builds a model.Package to send as GraphQL response, starting from id. +// The optional filter allows restricting output (on selection operations). +func (c *demoClient) buildPackageResponse(ctx context.Context, id string, filter *model.PkgSpec) (*model.Package, error) { + if filter != nil && filter.ID != nil && *filter.ID != id { + return nil, nil + } + + currentID := id + + pvl := []*model.PackageVersion{} + if versionNode, err := byIDkv[*pkgVersion](ctx, currentID, c); err == nil { + if filter != nil && noMatch(filter.Version, versionNode.Version) { + return nil, nil + } + if filter != nil && noMatch(filter.Subpath, versionNode.Subpath) { + return nil, nil + } + if filter != nil && noMatchQualifiers(filter, versionNode.Qualifiers) { + return nil, nil + } + pvl = append(pvl, &model.PackageVersion{ + ID: versionNode.ThisID, + Version: versionNode.Version, + Subpath: versionNode.Subpath, + Qualifiers: getCollectedPackageQualifiers(versionNode.Qualifiers), + }) + currentID = versionNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + pnl := []*model.PackageName{} + if nameNode, err := byIDkv[*pkgName](ctx, currentID, c); err == nil { + if filter != nil && noMatch(filter.Name, nameNode.Name) { + return nil, nil + } + pnl = append(pnl, &model.PackageName{ + ID: nameNode.ThisID, + Name: nameNode.Name, + Versions: pvl, + }) + currentID = nameNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + pnsl := []*model.PackageNamespace{} + if namespaceNode, err := byIDkv[*pkgNamespace](ctx, currentID, c); err == nil { + if filter != nil && noMatch(filter.Namespace, namespaceNode.Namespace) { + return nil, nil + } + pnsl = append(pnsl, &model.PackageNamespace{ + ID: namespaceNode.ThisID, + Namespace: namespaceNode.Namespace, + Names: pnl, + }) + currentID = namespaceNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + typeNode, err := byIDkv[*pkgType](ctx, currentID, c) + if err != nil { + if errors.Is(err, kv.NotFoundError) || errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("%w: ID does not match expected node type for package namespace", errNotFound) + } else { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + } + if filter != nil && noMatch(filter.Type, typeNode.Type) { + return nil, nil + } + p := model.Package{ + ID: typeNode.ThisID, + Type: typeNode.Type, + Namespaces: pnsl, + } + return &p, nil +} + +func (c *demoClient) getPackageNameFromInput(ctx context.Context, input model.PkgInputSpec) (*pkgName, error) { + inType := &pkgType{ + Type: input.Type, + } + pkgT, err := byKeykv[*pkgType](ctx, pkgTypeCol, inType.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package type \"%s\" not found", input.Type) + } + + inNS := &pkgNamespace{ + Parent: pkgT.ThisID, + Namespace: nilToEmpty(input.Namespace), + } + pkgNS, err := byKeykv[*pkgNamespace](ctx, pkgNSCol, inNS.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package namespace \"%s\" not found", nilToEmpty(input.Namespace)) + } + + inName := &pkgName{ + Parent: pkgNS.ThisID, + Name: input.Name, + } + pkgN, err := byKeykv[*pkgName](ctx, pkgNameCol, inName.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package name \"%s\" not found", input.Name) + } + + return pkgN, nil +} + +func (c *demoClient) getPackageVerFromInput(ctx context.Context, input model.PkgInputSpec) (*pkgVersion, error) { + pkgN, err := c.getPackageNameFromInput(ctx, input) + if err != nil { + return nil, gqlerror.Errorf("Package name \"%s\" not found", input.Name) + } + + inVer := &pkgVersion{ + Parent: pkgN.ThisID, + Version: nilToEmpty(input.Version), + Subpath: nilToEmpty(input.Subpath), + Qualifiers: getQualifiersFromInput(input.Qualifiers), + } + pkgVer, err := byKeykv[*pkgVersion](ctx, pkgVerCol, inVer.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("No package matches input") + } + return pkgVer, nil +} + +func (c *demoClient) getPackageNameOrVerFromInput(ctx context.Context, input model.PkgInputSpec, pkgMatchType model.MatchFlags) (pkgNameOrVersion, error) { + if pkgMatchType.Pkg == model.PkgMatchTypeAllVersions { + return c.getPackageNameFromInput(ctx, input) + } + return c.getPackageVerFromInput(ctx, input) +} + +func getCollectedPackageQualifiers(qualifierMap map[string]string) []*model.PackageQualifier { + qualifiers := []*model.PackageQualifier{} + for key, val := range qualifierMap { + qualifier := &model.PackageQualifier{ + Key: key, + Value: val, + } + qualifiers = append(qualifiers, qualifier) + + } + return qualifiers +} + +func getQualifiersFromInput(qualifiersSpec []*model.PackageQualifierInputSpec) map[string]string { + qualifiersMap := map[string]string{} + // if qualifiersSpec == nil { + // return qualifiersMap + // } + for _, kv := range qualifiersSpec { + if kv != nil { + qualifiersMap[kv.Key] = kv.Value + } + } + return qualifiersMap +} + +func getQualifiersFromFilter(qualifiersSpec []*model.PackageQualifierSpec) map[string]string { + qualifiersMap := map[string]string{} + if qualifiersSpec == nil { + return qualifiersMap + } + for _, kv := range qualifiersSpec { + qualifiersMap[kv.Key] = nilToEmpty(kv.Value) + } + return qualifiersMap +} + +func noMatchQualifiers(filter *model.PkgSpec, v map[string]string) bool { + // Allow matching on nodes with no qualifiers + if filter.MatchOnlyEmptyQualifiers != nil { + if *filter.MatchOnlyEmptyQualifiers && len(v) != 0 { + return true + } + } + if filter.Qualifiers != nil && len(filter.Qualifiers) > 0 { + filterQualifiers := getQualifiersFromFilter(filter.Qualifiers) + return !reflect.DeepEqual(v, filterQualifiers) + } + return false +} + +func (c *demoClient) findPackageVersion(ctx context.Context, filter *model.PkgSpec) ([]*pkgVersion, error) { + if filter == nil { + return nil, nil + } + if filter.ID != nil { + if pkgVer, err := byIDkv[*pkgVersion](ctx, *filter.ID, c); err == nil { + return []*pkgVersion{pkgVer}, nil + } else { // fixme check if err is not keyerror and bubble up if needed + return nil, nil + } + } + if filter.Type == nil || filter.Namespace != nil || filter.Name == nil || filter.Version == nil { // search all ver? + return nil, nil + } + + pkgN, err := c.exactPackageName(ctx, filter) + if err != nil { + return nil, nil + } + + var out []*pkgVersion + for _, vID := range pkgN.Versions { + pkgVer, err := byIDkv[*pkgVersion](ctx, vID, c) + if err != nil { + return nil, err + } + if *filter.Version != pkgVer.Version || + noMatch(filter.Subpath, pkgVer.Subpath) || + noMatchQualifiers(filter, pkgVer.Qualifiers) { + continue + } + out = append(out, pkgVer) + } + return out, nil +} + +func (c *demoClient) exactPackageName(ctx context.Context, filter *model.PkgSpec) (*pkgName, error) { + if filter == nil { + return nil, nil + } + if filter.ID != nil { + if pkgN, err := byIDkv[*pkgName](ctx, *filter.ID, c); err == nil { + return pkgN, nil + } else { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + } + if filter.Type == nil || filter.Namespace != nil || filter.Name == nil { + return nil, nil + } + inType := &pkgType{ + Type: *filter.Type, + } + pkgT, err := byKeykv[*pkgType](ctx, pkgTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + + inNS := &pkgNamespace{ + Parent: pkgT.ThisID, + Namespace: *filter.Namespace, + } + pkgNS, err := byKeykv[*pkgNamespace](ctx, pkgNSCol, inNS.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + + inName := &pkgName{ + Parent: pkgNS.ThisID, + Name: *filter.Name, + } + pkgN, err := byKeykv[*pkgName](ctx, pkgNameCol, inName.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + return pkgN, nil +} + +func (c *demoClient) matchPackages(ctx context.Context, filter []*model.PkgSpec, pkgs []string) bool { + pkgs = slices.Clone(pkgs) + pkgs = sortAndRemoveDups(pkgs) + + for _, pvSpec := range filter { + if pvSpec != nil { + if pvSpec.ID != nil { + // Check by ID if present + if !c.isIDPresent(*pvSpec.ID, pkgs) { + return false + } + } else { + // Otherwise match spec information + match := false + for _, pkgId := range pkgs { + id := pkgId + pkgVersion, err := byIDkv[*pkgVersion](ctx, id, c) + if err == nil { + if noMatch(pvSpec.Subpath, pkgVersion.Subpath) || noMatchQualifiers(pvSpec, pkgVersion.Qualifiers) || noMatch(pvSpec.Version, pkgVersion.Version) { + continue + } + id = pkgVersion.Parent + } + pkgName, err := byIDkv[*pkgName](ctx, id, c) + if err == nil { + if noMatch(pvSpec.Name, pkgName.Name) { + continue + } + id = pkgName.Parent + } + pkgNamespace, err := byIDkv[*pkgNamespace](ctx, id, c) + if err == nil { + if noMatch(pvSpec.Namespace, pkgNamespace.Namespace) { + continue + } + id = pkgNamespace.Parent + } + pkgType, err := byIDkv[*pkgType](ctx, id, c) + if err == nil { + if noMatch(pvSpec.Type, pkgType.Type) { + continue + } else { + match = true + break + } + } + } + if !match { + return false + } + } + } + } + return true +} diff --git a/pkg/assembler/backends/inmem/pkgEqual.go b/pkg/assembler/backends/keyvalue/pkgEqual.go similarity index 58% rename from pkg/assembler/backends/inmem/pkgEqual.go rename to pkg/assembler/backends/keyvalue/pkgEqual.go index 26f377e0dd..46dc71c04d 100644 --- a/pkg/assembler/backends/inmem/pkgEqual.go +++ b/pkg/assembler/backends/keyvalue/pkgEqual.go @@ -13,39 +13,47 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" + "fmt" "slices" - "strconv" + "strings" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) -type ( - pkgEqualList []*pkgEqualStruct - pkgEqualStruct struct { - id uint32 - pkgs []uint32 - justification string - origin string - collector string - } -) +type pkgEqualStruct struct { + ThisID string + Pkgs []string + Justification string + Origin string + Collector string +} -func (n *pkgEqualStruct) ID() uint32 { return n.id } +func (n *pkgEqualStruct) ID() string { return n.ThisID } +func (n *pkgEqualStruct) Key() string { + return strings.Join([]string{ + fmt.Sprint(n.Pkgs), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *pkgEqualStruct) Neighbors(allowedEdges edgeMap) []uint32 { +func (n *pkgEqualStruct) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgePkgEqualPackage] { - return n.pkgs + return n.Pkgs } - return []uint32{} + return nil } -func (n *pkgEqualStruct) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convPkgEqual(n) +func (n *pkgEqualStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convPkgEqual(ctx, n) } // Ingest PkgEqual @@ -62,15 +70,15 @@ func (c *demoClient) IngestPkgEquals(ctx context.Context, pkgs []*model.PkgInput return modelPkgEqualsIDs, nil } -func (c *demoClient) convPkgEqual(in *pkgEqualStruct) (*model.PkgEqual, error) { +func (c *demoClient) convPkgEqual(ctx context.Context, in *pkgEqualStruct) (*model.PkgEqual, error) { out := &model.PkgEqual{ - ID: nodeID(in.id), - Justification: in.justification, - Origin: in.origin, - Collector: in.collector, + ID: in.ThisID, + Justification: in.Justification, + Origin: in.Origin, + Collector: in.Collector, } - for _, id := range in.pkgs { - p, err := c.buildPackageResponse(id, nil) + for _, id := range in.Pkgs { + p, err := c.buildPackageResponse(ctx, id, nil) if err != nil { return nil, err } @@ -85,36 +93,35 @@ func (c *demoClient) IngestPkgEqual(ctx context.Context, pkg model.PkgInputSpec, func (c *demoClient) ingestPkgEqual(ctx context.Context, pkg model.PkgInputSpec, depPkg model.PkgInputSpec, pkgEqual model.PkgEqualInputSpec, readOnly bool) (*model.PkgEqual, error) { funcName := "IngestPkgEqual" + + in := &pkgEqualStruct{ + Justification: pkgEqual.Justification, + Origin: pkgEqual.Origin, + Collector: pkgEqual.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - pIDs := make([]uint32, 0, 2) + pIDs := make([]string, 0, 2) + ps := make([]*pkgVersion, 0, 2) for _, pi := range []model.PkgInputSpec{pkg, depPkg} { - pid, err := getPackageIDFromInput(c, pi, model.MatchFlags{Pkg: model.PkgMatchTypeSpecificVersion}) + p, err := c.getPackageVerFromInput(ctx, pi) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - pIDs = append(pIDs, pid) + ps = append(ps, p) + pIDs = append(pIDs, p.ThisID) } slices.Sort(pIDs) + in.Pkgs = pIDs - ps := make([]*pkgVersionNode, 0, 2) - for _, pID := range pIDs { - p, _ := byID[*pkgVersionNode](pID, c) - ps = append(ps, p) + out, err := byKeykv[*pkgEqualStruct](ctx, pkgEqCol, in.Key(), c) + if err == nil { + return c.convPkgEqual(ctx, out) } - - for _, id := range ps[0].pkgEquals { - cp, err := byID[*pkgEqualStruct](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if slices.Equal(cp.pkgs, pIDs) && - cp.justification == pkgEqual.Justification && - cp.origin == pkgEqual.Origin && - cp.collector == pkgEqual.Collector { - return c.convPkgEqual(cp) - } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } if readOnly { @@ -124,20 +131,20 @@ func (c *demoClient) ingestPkgEqual(ctx context.Context, pkg model.PkgInputSpec, return cp, err } - cp := &pkgEqualStruct{ - id: c.getNextID(), - pkgs: pIDs, - justification: pkgEqual.Justification, - origin: pkgEqual.Origin, - collector: pkgEqual.Collector, + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pkgEqCol, in); err != nil { + return nil, err } - c.index[cp.id] = cp for _, p := range ps { - p.setPkgEquals(cp.id) + if err := p.setPkgEquals(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := setkv(ctx, pkgEqCol, in, c); err != nil { + return nil, err } - c.pkgEquals = append(c.pkgEquals, cp) - return c.convPkgEqual(cp) + return c.convPkgEqual(ctx, in) } // Query PkgEqual @@ -147,51 +154,53 @@ func (c *demoClient) PkgEqual(ctx context.Context, filter *model.PkgEqualSpec) ( c.m.RLock() defer c.m.RUnlock() if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*pkgEqualStruct](id, c) + link, err := byIDkv[*pkgEqualStruct](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - pe, err := c.convPkgEqual(link) + pe, err := c.convPkgEqual(ctx, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.PkgEqual{pe}, nil } - var search []uint32 + var search []string for _, p := range filter.Packages { - pkgs, err := c.findPackageVersion(p) + pkgs, err := c.findPackageVersion(ctx, p) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } for _, pkg := range pkgs { - search = append(search, pkg.pkgEquals...) + search = append(search, pkg.PkgEquals...) } } var out []*model.PkgEqual if len(search) > 0 { for _, id := range search { - link, err := byID[*pkgEqualStruct](id, c) + link, err := byIDkv[*pkgEqualStruct](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addCPIfMatch(out, filter, link) + out, err = c.addCPIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.pkgEquals { - var err error - out, err = c.addCPIfMatch(out, filter, link) + peKeys, err := c.kv.Keys(ctx, pkgEqCol) + if err != nil { + return nil, err + } + for _, pek := range peKeys { + link, err := byKeykv[*pkgEqualStruct](ctx, pkgEqCol, pek, c) + if err != nil { + return nil, err + } + out, err = c.addCPIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -200,13 +209,13 @@ func (c *demoClient) PkgEqual(ctx context.Context, filter *model.PkgEqualSpec) ( return out, nil } -func (c *demoClient) addCPIfMatch(out []*model.PkgEqual, +func (c *demoClient) addCPIfMatch(ctx context.Context, out []*model.PkgEqual, filter *model.PkgEqualSpec, link *pkgEqualStruct) ( []*model.PkgEqual, error, ) { - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) { + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) { return out, nil } for _, ps := range filter.Packages { @@ -214,8 +223,8 @@ func (c *demoClient) addCPIfMatch(out []*model.PkgEqual, continue } found := false - for _, pid := range link.pkgs { - p, err := c.buildPackageResponse(pid, ps) + for _, pid := range link.Pkgs { + p, err := c.buildPackageResponse(ctx, pid, ps) if err != nil { return nil, err } @@ -227,7 +236,7 @@ func (c *demoClient) addCPIfMatch(out []*model.PkgEqual, return out, nil } } - pe, err := c.convPkgEqual(link) + pe, err := c.convPkgEqual(ctx, link) if err != nil { return nil, err } diff --git a/pkg/assembler/backends/inmem/pkgEqual_test.go b/pkg/assembler/backends/keyvalue/pkgEqual_test.go similarity index 95% rename from pkg/assembler/backends/inmem/pkgEqual_test.go rename to pkg/assembler/backends/keyvalue/pkgEqual_test.go index 6000cf65b3..e0d7c88c49 100644 --- a/pkg/assembler/backends/inmem/pkgEqual_test.go +++ b/pkg/assembler/backends/keyvalue/pkgEqual_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,8 +23,9 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" - "github.com/guacsec/guac/pkg/assembler/backends/inmem" + "github.com/guacsec/guac/pkg/assembler/backends/keyvalue" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -387,31 +388,6 @@ func TestPkgEqual(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query bad ID", - InPkg: []*model.PkgInputSpec{p1, p2, p3}, - Calls: []call{ - { - P1: p1, - P2: p2, - HE: &model.PkgEqualInputSpec{}, - }, - { - P1: p2, - P2: p3, - HE: &model.PkgEqualInputSpec{}, - }, - { - P1: p1, - P2: p3, - HE: &model.PkgEqualInputSpec{}, - }, - }, - Query: &model.PkgEqualSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -419,7 +395,8 @@ func TestPkgEqual(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -445,8 +422,8 @@ func TestPkgEqual(t *testing.T) { return } - inmem.MakeCanonicalPkgEqualSlice(got) - inmem.MakeCanonicalPkgEqualSlice(test.ExpHE) + keyvalue.MakeCanonicalPkgEqualSlice(got) + keyvalue.MakeCanonicalPkgEqualSlice(test.ExpHE) if diff := cmp.Diff(test.ExpHE, got, ignoreID); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) @@ -632,7 +609,8 @@ func TestIngestPkgEquals(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -656,8 +634,8 @@ func TestIngestPkgEquals(t *testing.T) { return } - inmem.MakeCanonicalPkgEqualSlice(got) - inmem.MakeCanonicalPkgEqualSlice(test.ExpHE) + keyvalue.MakeCanonicalPkgEqualSlice(got) + keyvalue.MakeCanonicalPkgEqualSlice(test.ExpHE) if diff := cmp.Diff(test.ExpHE, got, ignoreID); diff != "" { t.Errorf("Unexpected results. (-want +got):\n%s", diff) @@ -727,7 +705,8 @@ func TestPkgEqualNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/pkg_test.go b/pkg/assembler/backends/keyvalue/pkg_test.go similarity index 62% rename from pkg/assembler/backends/inmem/pkg_test.go rename to pkg/assembler/backends/keyvalue/pkg_test.go index d6cc3989b4..f891cdfc05 100644 --- a/pkg/assembler/backends/inmem/pkg_test.go +++ b/pkg/assembler/backends/keyvalue/pkg_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" @@ -26,396 +26,404 @@ import ( "github.com/guacsec/guac/pkg/assembler/graphql/model" ) -func Test_pkgNamespaceStruct_ID(t *testing.T) { +func Test_pkgType_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgNamespaceStruct{ - id: tt.id, + n := &pkgType{ + ThisID: tt.id, } if got := n.ID(); got != tt.want { - t.Errorf("pkgNamespaceStruct.ID() = %v, want %v", got, tt.want) + t.Errorf("pkgType.ID() = %v, want %v", got, tt.want) } }) } } -func Test_pkgNameStruct_ID(t *testing.T) { +func Test_pkgNamespace_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgNameStruct{ - id: tt.id, + n := &pkgNamespace{ + ThisID: tt.id, } if got := n.ID(); got != tt.want { - t.Errorf("pkgNameStruct.ID() = %v, want %v", got, tt.want) + t.Errorf("pkgNamespace.ID() = %v, want %v", got, tt.want) } }) } } -func Test_pkgVersionStruct_ID(t *testing.T) { +func Test_pkgName_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgVersionStruct{ - id: tt.id, + n := &pkgName{ + ThisID: tt.id, } if got := n.ID(); got != tt.want { - t.Errorf("pkgVersionStruct.ID() = %v, want %v", got, tt.want) + t.Errorf("pkgName.ID() = %v, want %v", got, tt.want) } }) } } -func Test_pkgVersionNode_ID(t *testing.T) { +func Test_pkgVersion_ID(t *testing.T) { tests := []struct { name string - id uint32 - want uint32 + id string + want string }{{ name: "getID", - id: 643, - want: 643, + id: "643", + want: "643", }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgVersionNode{ - id: tt.id, + n := &pkgVersion{ + ThisID: tt.id, } if got := n.ID(); got != tt.want { - t.Errorf("pkgVersionNode.ID() = %v, want %v", got, tt.want) + t.Errorf("pkgVersion.ID() = %v, want %v", got, tt.want) } }) } } -func Test_pkgNamespaceStruct_Neighbors(t *testing.T) { +func Test_PkgType_Neighbors(t *testing.T) { type fields struct { - id uint32 - namespaces pkgNamespaceMap + id string + namespaces []string } tests := []struct { name string fields fields - want []uint32 + want []string }{{ - name: "pkgNamespaceStruct Neighbors", + name: "PkgType Neighbors", fields: fields{ - id: uint32(23), - namespaces: pkgNamespaceMap{"test": &pkgNameStruct{id: uint32(24)}}, + id: "23", + namespaces: []string{"24"}, }, - want: []uint32{uint32(24)}, + want: []string{"24"}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgNamespaceStruct{ - id: tt.fields.id, - namespaces: tt.fields.namespaces, + n := &pkgType{ + ThisID: tt.fields.id, + Namespaces: tt.fields.namespaces, } if got := n.Neighbors(edgeMap{ model.EdgePackageTypePackageNamespace: true, }); !reflect.DeepEqual(got, tt.want) { - t.Errorf("pkgNamespaceStruct.Neighbors() = %v, want %v", got, tt.want) + t.Errorf("PkgType.Neighbors() = %v, want %v", got, tt.want) } }) } } -func Test_pkgNameStruct_Neighbors(t *testing.T) { +func Test_pkgNamespace_Neighbors(t *testing.T) { type fields struct { - id uint32 - parent uint32 + id string + parent string namespace string - names pkgNameMap + names []string } tests := []struct { name string fields fields - want []uint32 + want []string }{{ - name: "pkgNameStruct Neighbors", + name: "pkgNamespace Neighbors", fields: fields{ - id: uint32(23), - parent: uint32(22), + id: "23", + parent: "22", namespace: "test", - names: pkgNameMap{"test": &pkgVersionStruct{id: uint32(24)}}, + names: []string{"24"}, }, - want: []uint32{uint32(24), uint32(22)}, + want: []string{"24", "22"}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgNameStruct{ - id: tt.fields.id, - parent: tt.fields.parent, - namespace: tt.fields.namespace, - names: tt.fields.names, + n := &pkgNamespace{ + ThisID: tt.fields.id, + Parent: tt.fields.parent, + Namespace: tt.fields.namespace, + Names: tt.fields.names, } if got := n.Neighbors(edgeMap{ model.EdgePackageNamespacePackageType: true, model.EdgePackageNamespacePackageName: true, }); !reflect.DeepEqual(got, tt.want) { - t.Errorf("pkgNameStruct.Neighbors() = %v, want %v", got, tt.want) + t.Errorf("pkgNamespace.Neighbors() = %v, want %v", got, tt.want) } }) } } -func Test_pkgVersionStruct_Neighbors(t *testing.T) { +func Test_pkgName_Neighbors(t *testing.T) { type fields struct { - id uint32 - parent uint32 - versions pkgVersionMap - srcMapLinks []uint32 - isDependencyLinks []uint32 - badLinks []uint32 - goodLinks []uint32 + id string + parent string + versions []string + srcMapLinks []string + isDependencyLinks []string + badLinks []string + goodLinks []string } tests := []struct { name string allowedEdges edgeMap fields fields - want []uint32 - }{{ - name: "packageNamespace", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - srcMapLinks: []uint32{343, 546}, - }, - allowedEdges: edgeMap{model.EdgePackageNamePackageNamespace: true}, - want: []uint32{22}, - }, { - name: "packageVersion", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - srcMapLinks: []uint32{343, 546}, + want []string + }{ + { + name: "packageNamespace", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + srcMapLinks: []string{"343", "546"}, + }, + allowedEdges: edgeMap{model.EdgePackageNamePackageNamespace: true}, + want: []string{"22"}, }, - allowedEdges: edgeMap{model.EdgePackageNamePackageVersion: true}, - want: []uint32{24}, - }, { - name: "srcMapLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - srcMapLinks: []uint32{343, 546}, + { + name: "packageVersion", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + srcMapLinks: []string{"343", "546"}, + }, + allowedEdges: edgeMap{model.EdgePackageNamePackageVersion: true}, + want: []string{"24"}, }, - allowedEdges: edgeMap{model.EdgePackageHasSourceAt: true}, - want: []uint32{343, 546}, - }, { - name: "isDependencyLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - isDependencyLinks: []uint32{2324, 1234}, + { + name: "srcMapLinks", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + srcMapLinks: []string{"343", "546"}, + }, + allowedEdges: edgeMap{model.EdgePackageHasSourceAt: true}, + want: []string{"343", "546"}, }, - allowedEdges: edgeMap{model.EdgePackageIsDependency: true}, - want: []uint32{2324, 1234}, - }, { - name: "badLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - badLinks: []uint32{445, 1232244}, + { + name: "isDependencyLinks", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + isDependencyLinks: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageIsDependency: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyBad: true}, - want: []uint32{445, 1232244}, - }, { - name: "goodLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - goodLinks: []uint32{987, 9876}, + { + name: "badLinks", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + badLinks: []string{"445", "1232244"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyBad: true}, + want: []string{"445", "1232244"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyGood: true}, - want: []uint32{987, 9876}, - }, { - name: "goodLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - versions: pkgVersionMap{"digest-a": &pkgVersionNode{id: uint32(24)}}, - goodLinks: []uint32{987, 9876}, + { + name: "goodLinks", + fields: fields{ + id: "23", + parent: "22", + versions: []string{"24"}, + goodLinks: []string{"987", "9876"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyGood: true}, + want: []string{"987", "9876"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyGood: true}, - want: []uint32{987, 9876}, - }} + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgVersionStruct{ - id: tt.fields.id, - parent: tt.fields.parent, - versions: tt.fields.versions, - srcMapLinks: tt.fields.srcMapLinks, - isDependencyLinks: tt.fields.isDependencyLinks, - badLinks: tt.fields.badLinks, - goodLinks: tt.fields.goodLinks, + n := &pkgName{ + ThisID: tt.fields.id, + Parent: tt.fields.parent, + Versions: tt.fields.versions, + SrcMapLinks: tt.fields.srcMapLinks, + IsDependencyLinks: tt.fields.isDependencyLinks, + BadLinks: tt.fields.badLinks, + GoodLinks: tt.fields.goodLinks, } if got := n.Neighbors(tt.allowedEdges); !reflect.DeepEqual(got, tt.want) { - t.Errorf("pkgVersionStruct.Neighbors() = %v, want %v", got, tt.want) + t.Errorf("pkgName.Neighbors() = %v, want %v", got, tt.want) } }) } } -func Test_pkgVersionNode_Neighbors(t *testing.T) { +func Test_pkgVersion_Neighbors(t *testing.T) { type fields struct { - id uint32 - parent uint32 - srcMapLinks []uint32 - isDependencyLinks []uint32 - occurrences []uint32 - certifyVulnLinks []uint32 - hasSBOMs []uint32 - vexLinks []uint32 - badLinks []uint32 - goodLinks []uint32 - pkgEquals []uint32 + id string + parent string + srcMapLinks []string + isDependencyLinks []string + occurrences []string + certifyVulnLinks []string + hasSBOMs []string + vexLinks []string + badLinks []string + goodLinks []string + pkgEquals []string } tests := []struct { name string allowedEdges edgeMap fields fields - want []uint32 - }{{ - name: "packageName", - fields: fields{ - id: uint32(23), - parent: uint32(22), - srcMapLinks: []uint32{343, 546}, + want []string + }{ + { + name: "packageName", + fields: fields{ + id: "23", + parent: "22", + srcMapLinks: []string{"343", "546"}, + }, + allowedEdges: edgeMap{model.EdgePackageVersionPackageName: true}, + want: []string{"22"}, }, - allowedEdges: edgeMap{model.EdgePackageVersionPackageName: true}, - want: []uint32{22}, - }, { - name: "srcMapLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - srcMapLinks: []uint32{343, 546}, + { + name: "srcMapLinks", + fields: fields{ + id: "23", + parent: "22", + srcMapLinks: []string{"343", "546"}, + }, + allowedEdges: edgeMap{model.EdgePackageHasSourceAt: true}, + want: []string{"343", "546"}, }, - allowedEdges: edgeMap{model.EdgePackageHasSourceAt: true}, - want: []uint32{343, 546}, - }, { - name: "isDependencyLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - isDependencyLinks: []uint32{2324, 1234}, + { + name: "isDependencyLinks", + fields: fields{ + id: "23", + parent: "22", + isDependencyLinks: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageIsDependency: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageIsDependency: true}, - want: []uint32{2324, 1234}, - }, { - name: "occurrences", - fields: fields{ - id: uint32(23), - parent: uint32(22), - occurrences: []uint32{2324, 1234}, + { + name: "occurrences", + fields: fields{ + id: "23", + parent: "22", + occurrences: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageIsOccurrence: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageIsOccurrence: true}, - want: []uint32{2324, 1234}, - }, { - name: "certifyVulnLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - certifyVulnLinks: []uint32{2324, 1234}, + { + name: "certifyVulnLinks", + fields: fields{ + id: "23", + parent: "22", + certifyVulnLinks: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyVuln: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyVuln: true}, - want: []uint32{2324, 1234}, - }, { - name: "hasSBOMs", - fields: fields{ - id: uint32(23), - parent: uint32(22), - hasSBOMs: []uint32{2324, 1234}, + { + name: "hasSBOMs", + fields: fields{ + id: "23", + parent: "22", + hasSBOMs: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageHasSbom: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageHasSbom: true}, - want: []uint32{2324, 1234}, - }, { - name: "vexLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - vexLinks: []uint32{2324, 1234}, + { + name: "vexLinks", + fields: fields{ + id: "23", + parent: "22", + vexLinks: []string{"2324", "1234"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyVexStatement: true}, + want: []string{"2324", "1234"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyVexStatement: true}, - want: []uint32{2324, 1234}, - }, { - name: "badLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - badLinks: []uint32{445, 1232244}, + { + name: "badLinks", + fields: fields{ + id: "23", + parent: "22", + badLinks: []string{"445", "1232244"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyBad: true}, + want: []string{"445", "1232244"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyBad: true}, - want: []uint32{445, 1232244}, - }, { - name: "goodLinks", - fields: fields{ - id: uint32(23), - parent: uint32(22), - goodLinks: []uint32{987, 9876}, + { + name: "goodLinks", + fields: fields{ + id: "23", + parent: "22", + goodLinks: []string{"987", "9876"}, + }, + allowedEdges: edgeMap{model.EdgePackageCertifyGood: true}, + want: []string{"987", "9876"}, }, - allowedEdges: edgeMap{model.EdgePackageCertifyGood: true}, - want: []uint32{987, 9876}, - }, { - name: "pkgEquals", - fields: fields{ - id: uint32(23), - parent: uint32(22), - pkgEquals: []uint32{987, 9876}, + { + name: "pkgEquals", + fields: fields{ + id: "23", + parent: "22", + pkgEquals: []string{"987", "9876"}, + }, + allowedEdges: edgeMap{model.EdgePackagePkgEqual: true}, + want: []string{"987", "9876"}, }, - allowedEdges: edgeMap{model.EdgePackagePkgEqual: true}, - want: []uint32{987, 9876}, - }} + } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - n := &pkgVersionNode{ - id: tt.fields.id, - parent: tt.fields.parent, - srcMapLinks: tt.fields.srcMapLinks, - isDependencyLinks: tt.fields.isDependencyLinks, - occurrences: tt.fields.occurrences, - certifyVulnLinks: tt.fields.certifyVulnLinks, - hasSBOMs: tt.fields.hasSBOMs, - vexLinks: tt.fields.vexLinks, - badLinks: tt.fields.badLinks, - goodLinks: tt.fields.goodLinks, - pkgEquals: tt.fields.pkgEquals, + n := &pkgVersion{ + ThisID: tt.fields.id, + Parent: tt.fields.parent, + SrcMapLinks: tt.fields.srcMapLinks, + IsDependencyLinks: tt.fields.isDependencyLinks, + Occurrences: tt.fields.occurrences, + CertifyVulnLinks: tt.fields.certifyVulnLinks, + HasSBOMs: tt.fields.hasSBOMs, + VexLinks: tt.fields.vexLinks, + BadLinks: tt.fields.badLinks, + GoodLinks: tt.fields.goodLinks, + PkgEquals: tt.fields.pkgEquals, } if got := n.Neighbors(tt.allowedEdges); !reflect.DeepEqual(got, tt.want) { - t.Errorf("pkgVersionNode.Neighbors() = %v, want %v", got, tt.want) + t.Errorf("pkgVersion.Neighbors() = %v, want %v", got, tt.want) } }) } @@ -560,10 +568,7 @@ func Test_demoClient_Packages(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - packages: pkgTypeMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) ingestedPkg, err := c.IngestPackage(ctx, *tt.pkgInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestPackage() error = %v, wantErr %v", err, tt.wantErr) @@ -603,10 +608,7 @@ func Test_demoClient_IngestPackages(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - packages: pkgTypeMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) got, err := c.IngestPackages(ctx, tt.pkgInputs) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestPackages() error = %v, wantErr %v", err, tt.wantErr) @@ -843,10 +845,7 @@ func Test_IngestingVersions(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - packages: pkgTypeMap{}, - index: indexType{}, - } + c, _ := getBackend(ctx, nil) _, err := c.IngestPackages(ctx, tt.pkgInputs) if err != nil { t.Errorf("Unexpected demoClient.IngestPackages() error = %v, ", err) diff --git a/pkg/assembler/backends/inmem/pointOfContact.go b/pkg/assembler/backends/keyvalue/pointOfContact.go similarity index 50% rename from pkg/assembler/backends/inmem/pointOfContact.go rename to pkg/assembler/backends/keyvalue/pointOfContact.go index 4e046a22fa..412d215caa 100644 --- a/pkg/assembler/backends/inmem/pointOfContact.go +++ b/pkg/assembler/backends/keyvalue/pointOfContact.go @@ -13,51 +13,65 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" - "strconv" + "errors" + "strings" "time" "github.com/vektah/gqlparser/v2/gqlerror" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" ) // Internal data: link that a package/source/artifact is good -type pointOfContactList []*pointOfContactLink type pointOfContactLink struct { - id uint32 - packageID uint32 - artifactID uint32 - sourceID uint32 - email string - info string - since time.Time - justification string - origin string - collector string + ThisID string + PackageID string + ArtifactID string + SourceID string + Email string + Info string + Since time.Time + Justification string + Origin string + Collector string } -func (n *pointOfContactLink) ID() uint32 { return n.id } +func (n *pointOfContactLink) ID() string { return n.ThisID } +func (n *pointOfContactLink) Key() string { + return strings.Join([]string{ + n.PackageID, + n.ArtifactID, + n.SourceID, + n.Email, + n.Info, + timeKey(n.Since), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *pointOfContactLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 1) - if n.packageID != 0 && allowedEdges[model.EdgePointOfContactPackage] { - out = append(out, n.packageID) +func (n *pointOfContactLink) Neighbors(allowedEdges edgeMap) []string { + out := make([]string, 0, 1) + if n.PackageID != "" && allowedEdges[model.EdgePointOfContactPackage] { + out = append(out, n.PackageID) } - if n.artifactID != 0 && allowedEdges[model.EdgePointOfContactArtifact] { - out = append(out, n.artifactID) + if n.ArtifactID != "" && allowedEdges[model.EdgePointOfContactArtifact] { + out = append(out, n.ArtifactID) } - if n.sourceID != 0 && allowedEdges[model.EdgePointOfContactSource] { - out = append(out, n.sourceID) + if n.SourceID != "" && allowedEdges[model.EdgePointOfContactSource] { + out = append(out, n.SourceID) } return out } -func (n *pointOfContactLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildPointOfContact(n, nil, true) +func (n *pointOfContactLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildPointOfContact(ctx, n, nil, true) } // Ingest PointOfContact @@ -99,120 +113,84 @@ func (c *demoClient) IngestPointOfContact(ctx context.Context, subject model.Pac func (c *demoClient) ingestPointOfContact(ctx context.Context, subject model.PackageSourceOrArtifactInput, pkgMatchType *model.MatchFlags, pointOfContact model.PointOfContactInputSpec, readOnly bool) (*model.PointOfContact, error) { funcName := "IngestPointOfContact" + in := &pointOfContactLink{ + Email: pointOfContact.Email, + Info: pointOfContact.Info, + Since: pointOfContact.Since.UTC(), + Justification: pointOfContact.Justification, + Origin: pointOfContact.Origin, + Collector: pointOfContact.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var packageID uint32 var foundPkgNameorVersionNode pkgNameOrVersion - var artifactID uint32 var foundArtStrct *artStruct - var sourceID uint32 var srcName *srcNameNode - searchIDs := []uint32{} if subject.Package != nil { var err error - packageID, err = getPackageIDFromInput(c, *subject.Package, *pkgMatchType) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - foundPkgNameorVersionNode, err = byID[pkgNameOrVersion](packageID, c) + foundPkgNameorVersionNode, err = c.getPackageNameOrVerFromInput(ctx, *subject.Package, *pkgMatchType) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - searchIDs = append(searchIDs, foundPkgNameorVersionNode.getPointOfContactLinks()...) + in.PackageID = foundPkgNameorVersionNode.ID() } else if subject.Artifact != nil { var err error - artifactID, err = getArtifactIDFromInput(c, *subject.Artifact) + foundArtStrct, err = c.artifactByInput(ctx, subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - foundArtStrct, err = byID[*artStruct](artifactID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = append(searchIDs, foundArtStrct.pointOfContactLinks...) + in.ArtifactID = foundArtStrct.ThisID } else { var err error - sourceID, err = getSourceIDFromInput(c, *subject.Source) + srcName, err = c.getSourceNameFromInput(ctx, *subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %s", funcName, err) } - srcName, err = byID[*srcNameNode](sourceID, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - searchIDs = append(searchIDs, srcName.pointOfContactLinks...) + in.SourceID = srcName.ThisID } - // Don't insert duplicates - duplicate := false - collectedLink := pointOfContactLink{} - for _, id := range searchIDs { - v, err := byID[*pointOfContactLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %s", funcName, err) - } - subjectMatch := false - if packageID != 0 && packageID == v.packageID { - subjectMatch = true - } - if artifactID != 0 && artifactID == v.artifactID { - subjectMatch = true - } - if sourceID != 0 && sourceID == v.sourceID { - subjectMatch = true - } - if subjectMatch && pointOfContact.Justification == v.justification && - pointOfContact.Email == v.email && pointOfContact.Info == v.info && - pointOfContact.Since.Equal(v.since) && - pointOfContact.Origin == v.origin && pointOfContact.Collector == v.collector { - - collectedLink = *v - duplicate = true - break - } + out, err := byKeykv[*pointOfContactLink](ctx, pocCol, in.Key(), c) + if err == nil { + return c.buildPointOfContact(ctx, out, nil, true) } - if !duplicate { - if readOnly { - c.m.RUnlock() - b, err := c.ingestPointOfContact(ctx, subject, pkgMatchType, pointOfContact, false) - c.m.RLock() // relock so that defer unlock does not panic - return b, err - } - // store the link - collectedLink = pointOfContactLink{ - id: c.getNextID(), - packageID: packageID, - artifactID: artifactID, - sourceID: sourceID, - email: pointOfContact.Email, - info: pointOfContact.Info, - since: pointOfContact.Since, - justification: pointOfContact.Justification, - origin: pointOfContact.Origin, - collector: pointOfContact.Collector, - } - c.index[collectedLink.id] = &collectedLink - c.pointOfContacts = append(c.pointOfContacts, &collectedLink) - // set the backlinks - if packageID != 0 { - foundPkgNameorVersionNode.setPointOfContactLinks(collectedLink.id) + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + + if readOnly { + c.m.RUnlock() + b, err := c.ingestPointOfContact(ctx, subject, pkgMatchType, pointOfContact, false) + c.m.RLock() // relock so that defer unlock does not panic + return b, err + } + + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, pocCol, in); err != nil { + return nil, err + } + + if foundPkgNameorVersionNode != nil { + if err := foundPkgNameorVersionNode.setPointOfContactLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if artifactID != 0 { - foundArtStrct.setPointOfContactLinks(collectedLink.id) + } + if foundArtStrct != nil { + if err := foundArtStrct.setPointOfContactLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - if sourceID != 0 { - srcName.setPointOfContactLinks(collectedLink.id) + } + if srcName != nil { + if err := srcName.setPointOfContactLinks(ctx, in.ThisID, c); err != nil { + return nil, err } - } - - // build return GraphQL type - builtPointOfContact, err := c.buildPointOfContact(&collectedLink, nil, true) - if err != nil { + if err := setkv(ctx, pocCol, in, c); err != nil { return nil, err } - return builtPointOfContact, nil + + return c.buildPointOfContact(ctx, in, nil, true) } // Query PointOfContact @@ -223,17 +201,12 @@ func (c *demoClient) PointOfContact(ctx context.Context, filter *model.PointOfCo defer c.m.RUnlock() if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*pointOfContactLink](id, c) + link, err := byIDkv[*pointOfContactLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } - found, err := c.buildPointOfContact(link, filter, true) + found, err := c.buildPointOfContact(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -242,25 +215,25 @@ func (c *demoClient) PointOfContact(ctx context.Context, filter *model.PointOfCo // Cant really search for an exact Pkg, as these can be linked to either // names or versions, and version could be empty. - var search []uint32 + var search []string foundOne := false if filter != nil && filter.Subject != nil && filter.Subject.Artifact != nil { - exactArtifact, err := c.artifactExact(filter.Subject.Artifact) + exactArtifact, err := c.artifactExact(ctx, filter.Subject.Artifact) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactArtifact != nil { - search = append(search, exactArtifact.pointOfContactLinks...) + search = append(search, exactArtifact.PointOfContactLinks...) foundOne = true } } if !foundOne && filter != nil && filter.Subject != nil && filter.Subject.Source != nil { - exactSource, err := c.exactSource(filter.Subject.Source) + exactSource, err := c.exactSource(ctx, filter.Subject.Source) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactSource != nil { - search = append(search, exactSource.pointOfContactLinks...) + search = append(search, exactSource.PointOfContactLinks...) foundOne = true } } @@ -268,19 +241,26 @@ func (c *demoClient) PointOfContact(ctx context.Context, filter *model.PointOfCo var out []*model.PointOfContact if foundOne { for _, id := range search { - link, err := byID[*pointOfContactLink](id, c) + link, err := byIDkv[*pointOfContactLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addPOCIfMatch(out, filter, link) + out, err = c.addPOCIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.pointOfContacts { - var err error - out, err = c.addPOCIfMatch(out, filter, link) + pocKeys, err := c.kv.Keys(ctx, pocCol) + if err != nil { + return nil, err + } + for _, pk := range pocKeys { + link, err := byKeykv[*pointOfContactLink](ctx, pocCol, pk, c) + if err != nil { + return nil, err + } + out, err = c.addPOCIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -289,30 +269,30 @@ func (c *demoClient) PointOfContact(ctx context.Context, filter *model.PointOfCo return out, nil } -func (c *demoClient) addPOCIfMatch(out []*model.PointOfContact, filter *model.PointOfContactSpec, link *pointOfContactLink) ( +func (c *demoClient) addPOCIfMatch(ctx context.Context, out []*model.PointOfContact, filter *model.PointOfContactSpec, link *pointOfContactLink) ( []*model.PointOfContact, error) { - if filter != nil && noMatch(filter.Justification, link.justification) { + if filter != nil && noMatch(filter.Justification, link.Justification) { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - if filter != nil && noMatch(filter.Email, link.email) { + if filter != nil && noMatch(filter.Email, link.Email) { return out, nil } - if filter != nil && noMatch(filter.Info, link.info) { + if filter != nil && noMatch(filter.Info, link.Info) { return out, nil } // no match if filter time since is after the timestamp - if filter != nil && filter.Since != nil && filter.Since.After(link.since) { + if filter != nil && filter.Since != nil && filter.Since.After(link.Since) { return out, nil } - found, err := c.buildPointOfContact(link, filter, false) + found, err := c.buildPointOfContact(ctx, link, filter, false) if err != nil { return nil, err } @@ -322,45 +302,45 @@ func (c *demoClient) addPOCIfMatch(out []*model.PointOfContact, filter *model.Po return append(out, found), nil } -func (c *demoClient) buildPointOfContact(link *pointOfContactLink, filter *model.PointOfContactSpec, ingestOrIDProvided bool) (*model.PointOfContact, error) { +func (c *demoClient) buildPointOfContact(ctx context.Context, link *pointOfContactLink, filter *model.PointOfContactSpec, ingestOrIDProvided bool) (*model.PointOfContact, error) { var p *model.Package var a *model.Artifact var s *model.Source var err error if filter != nil && filter.Subject != nil { - if filter.Subject.Package != nil && link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, filter.Subject.Package) + if filter.Subject.Package != nil && link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, filter.Subject.Package) if err != nil { return nil, err } } - if filter.Subject.Artifact != nil && link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, filter.Subject.Artifact) + if filter.Subject.Artifact != nil && link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, filter.Subject.Artifact) if err != nil { return nil, err } } - if filter.Subject.Source != nil && link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, filter.Subject.Source) + if filter.Subject.Source != nil && link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, filter.Subject.Source) if err != nil { return nil, err } } } else { - if link.packageID != 0 { - p, err = c.buildPackageResponse(link.packageID, nil) + if link.PackageID != "" { + p, err = c.buildPackageResponse(ctx, link.PackageID, nil) if err != nil { return nil, err } } - if link.artifactID != 0 { - a, err = c.buildArtifactResponse(link.artifactID, nil) + if link.ArtifactID != "" { + a, err = c.buildArtifactResponse(ctx, link.ArtifactID, nil) if err != nil { return nil, err } } - if link.sourceID != 0 { - s, err = c.buildSourceResponse(link.sourceID, nil) + if link.SourceID != "" { + s, err = c.buildSourceResponse(ctx, link.SourceID, nil) if err != nil { return nil, err } @@ -368,7 +348,7 @@ func (c *demoClient) buildPointOfContact(link *pointOfContactLink, filter *model } var subj model.PackageSourceOrArtifact - if link.packageID != 0 { + if link.PackageID != "" { if p == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve package via packageID") } else if p == nil && !ingestOrIDProvided { @@ -376,7 +356,7 @@ func (c *demoClient) buildPointOfContact(link *pointOfContactLink, filter *model } subj = p } - if link.artifactID != 0 { + if link.ArtifactID != "" { if a == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve artifact via artifactID") } else if a == nil && !ingestOrIDProvided { @@ -384,7 +364,7 @@ func (c *demoClient) buildPointOfContact(link *pointOfContactLink, filter *model } subj = a } - if link.sourceID != 0 { + if link.SourceID != "" { if s == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve source via sourceID") } else if s == nil && !ingestOrIDProvided { @@ -394,14 +374,14 @@ func (c *demoClient) buildPointOfContact(link *pointOfContactLink, filter *model } pointOfContact := model.PointOfContact{ - ID: nodeID(link.id), + ID: link.ThisID, Subject: subj, - Email: link.email, - Info: link.info, - Since: link.since, - Justification: link.justification, - Origin: link.origin, - Collector: link.collector, + Email: link.Email, + Info: link.Info, + Since: link.Since, + Justification: link.Justification, + Origin: link.Origin, + Collector: link.Collector, } return &pointOfContact, nil } diff --git a/pkg/assembler/backends/inmem/pointOfContact_test.go b/pkg/assembler/backends/keyvalue/pointOfContact_test.go similarity index 98% rename from pkg/assembler/backends/inmem/pointOfContact_test.go rename to pkg/assembler/backends/keyvalue/pointOfContact_test.go index 8bf6bf2514..317f7ac1f4 100644 --- a/pkg/assembler/backends/inmem/pointOfContact_test.go +++ b/pkg/assembler/backends/keyvalue/pointOfContact_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -545,11 +546,11 @@ func TestPointOfContact(t *testing.T) { }, ExpHM: []*model.PointOfContact{ { - Subject: p2out, + Subject: p1outName, Justification: "test justification", }, { - Subject: p1outName, + Subject: p2out, Justification: "test justification", }, }, @@ -599,24 +600,6 @@ func TestPointOfContact(t *testing.T) { }, ExpIngestErr: true, }, - { - Name: "Query good ID", - InSrc: []*model.SourceInputSpec{s1}, - Calls: []call{ - { - Sub: model.PackageSourceOrArtifactInput{ - Source: s1, - }, - HM: &model.PointOfContactInputSpec{ - Justification: "test justification", - }, - }, - }, - Query: &model.PointOfContactSpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -624,7 +607,8 @@ func TestPointOfContact(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -920,7 +904,8 @@ func TestIngestPointOfContacts(t *testing.T) { for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1049,7 +1034,8 @@ func TestPointOfContactNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/search.go b/pkg/assembler/backends/keyvalue/search.go similarity index 98% rename from pkg/assembler/backends/inmem/search.go rename to pkg/assembler/backends/keyvalue/search.go index 4aaa1f30ca..4a8bffaa26 100644 --- a/pkg/assembler/backends/inmem/search.go +++ b/pkg/assembler/backends/keyvalue/search.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" diff --git a/pkg/assembler/backends/keyvalue/src.go b/pkg/assembler/backends/keyvalue/src.go new file mode 100644 index 0000000000..2a29e82b78 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/src.go @@ -0,0 +1,605 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +// Internal data: Sources +type srcType struct { + ThisID string + Type string + Namespaces []string +} +type srcNamespace struct { + ThisID string + Parent string + Namespace string + Names []string +} +type srcNameNode struct { + ThisID string + Parent string + Name string + Tag string + Commit string + SrcMapLinks []string + ScorecardLinks []string + Occurrences []string + BadLinks []string + GoodLinks []string + HasMetadataLinks []string + PointOfContactLinks []string + CertifyLegals []string +} + +func (n *srcType) ID() string { return n.ThisID } +func (n *srcNamespace) ID() string { return n.ThisID } +func (n *srcNameNode) ID() string { return n.ThisID } + +func (n *srcType) Key() string { + return n.Type +} + +func (n *srcNamespace) Key() string { + return strings.Join([]string{ + n.Parent, + n.Namespace, + }, ":") +} + +func (n *srcNameNode) Key() string { + return strings.Join([]string{ + n.Parent, + n.Name, + n.Tag, + n.Commit, + }, ":") +} + +func (n *srcType) Neighbors(allowedEdges edgeMap) []string { + if allowedEdges[model.EdgeSourceTypeSourceNamespace] { + return n.Namespaces + } + return nil +} +func (n *srcNamespace) Neighbors(allowedEdges edgeMap) []string { + var out []string + if allowedEdges[model.EdgeSourceNamespaceSourceName] { + out = append(out, n.Names...) + } + if allowedEdges[model.EdgeSourceNamespaceSourceType] { + out = append(out, n.Parent) + } + return out +} +func (n *srcNameNode) Neighbors(allowedEdges edgeMap) []string { + var out []string + + if allowedEdges[model.EdgeSourceNameSourceNamespace] { + out = append(out, n.Parent) + } + if allowedEdges[model.EdgeSourceHasSourceAt] { + out = append(out, n.SrcMapLinks...) + } + if allowedEdges[model.EdgeSourceCertifyScorecard] { + out = append(out, n.ScorecardLinks...) + } + if allowedEdges[model.EdgeSourceIsOccurrence] { + out = append(out, n.Occurrences...) + } + if allowedEdges[model.EdgeSourceCertifyBad] { + out = append(out, n.BadLinks...) + } + if allowedEdges[model.EdgeSourceCertifyGood] { + out = append(out, n.GoodLinks...) + } + if allowedEdges[model.EdgeSourceHasMetadata] { + out = append(out, n.HasMetadataLinks...) + } + if allowedEdges[model.EdgeSourcePointOfContact] { + out = append(out, n.PointOfContactLinks...) + } + if allowedEdges[model.EdgeSourceCertifyLegal] { + out = append(out, n.CertifyLegals...) + } + + return out +} + +func (n *srcType) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildSourceResponse(ctx, n.ThisID, nil) +} +func (n *srcNamespace) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildSourceResponse(ctx, n.ThisID, nil) +} +func (n *srcNameNode) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildSourceResponse(ctx, n.ThisID, nil) +} + +func (p *srcNameNode) setSrcMapLinks(ctx context.Context, id string, c *demoClient) error { + p.SrcMapLinks = append(p.SrcMapLinks, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setScorecardLinks(ctx context.Context, id string, c *demoClient) error { + p.ScorecardLinks = append(p.ScorecardLinks, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setOccurrenceLinks(ctx context.Context, id string, c *demoClient) error { + p.Occurrences = append(p.Occurrences, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setCertifyBadLinks(ctx context.Context, id string, c *demoClient) error { + p.BadLinks = append(p.BadLinks, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setCertifyGoodLinks(ctx context.Context, id string, c *demoClient) error { + p.GoodLinks = append(p.GoodLinks, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setCertifyLegals(ctx context.Context, id string, c *demoClient) error { + p.CertifyLegals = append(p.CertifyLegals, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setHasMetadataLinks(ctx context.Context, id string, c *demoClient) error { + p.HasMetadataLinks = append(p.HasMetadataLinks, id) + return setkv(ctx, srcNameCol, p, c) +} +func (p *srcNameNode) setPointOfContactLinks(ctx context.Context, id string, c *demoClient) error { + p.PointOfContactLinks = append(p.PointOfContactLinks, id) + return setkv(ctx, srcNameCol, p, c) +} + +func (n *srcType) addNamespace(ctx context.Context, ns string, c *demoClient) error { + n.Namespaces = append(n.Namespaces, ns) + return setkv(ctx, srcTypeCol, n, c) +} + +func (n *srcNamespace) addName(ctx context.Context, name string, c *demoClient) error { + n.Names = append(n.Names, name) + return setkv(ctx, srcNSCol, n, c) +} + +// Ingest Source + +func (c *demoClient) IngestSources(ctx context.Context, sources []*model.SourceInputSpec) ([]*model.Source, error) { + var modelSources []*model.Source + for _, src := range sources { + modelSrc, err := c.IngestSource(ctx, *src) + if err != nil { + return nil, gqlerror.Errorf("IngestSources failed with err: %v", err) + } + modelSources = append(modelSources, modelSrc) + } + return modelSources, nil +} + +func (c *demoClient) IngestSource(ctx context.Context, input model.SourceInputSpec) (*model.Source, error) { + inType := &srcType{ + Type: input.Type, + } + c.m.RLock() + outType, err := byKeykv[*srcType](ctx, srcTypeCol, inType.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outType, err = byKeykv[*srcType](ctx, srcTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inType.ThisID = c.getNextID() + if err := c.addToIndex(ctx, srcTypeCol, inType); err != nil { + return nil, err + } + if err := setkv(ctx, srcTypeCol, inType, c); err != nil { + return nil, err + } + outType = inType + } + c.m.Unlock() + } + + inNamespace := &srcNamespace{ + Parent: outType.ThisID, + Namespace: input.Namespace, + } + c.m.RLock() + outNamespace, err := byKeykv[*srcNamespace](ctx, srcNSCol, inNamespace.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outNamespace, err = byKeykv[*srcNamespace](ctx, srcNSCol, inNamespace.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inNamespace.ThisID = c.getNextID() + if err := c.addToIndex(ctx, srcNSCol, inNamespace); err != nil { + return nil, err + } + if err := setkv(ctx, srcNSCol, inNamespace, c); err != nil { + return nil, err + } + if err := outType.addNamespace(ctx, inNamespace.ThisID, c); err != nil { + return nil, err + } + outNamespace = inNamespace + } + c.m.Unlock() + } + + inName := &srcNameNode{ + Parent: outNamespace.ThisID, + Name: input.Name, + Tag: nilToEmpty(input.Tag), + Commit: nilToEmpty(input.Commit), + } + c.m.RLock() + outName, err := byKeykv[*srcNameNode](ctx, srcNameCol, inName.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outName, err = byKeykv[*srcNameNode](ctx, srcNameCol, inName.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inName.ThisID = c.getNextID() + if err := c.addToIndex(ctx, srcNameCol, inName); err != nil { + return nil, err + } + if err := setkv(ctx, srcNameCol, inName, c); err != nil { + return nil, err + } + if err := outNamespace.addName(ctx, inName.ThisID, c); err != nil { + return nil, err + } + outName = inName + } + c.m.Unlock() + } + + // build return GraphQL type + c.m.RLock() + defer c.m.RUnlock() + return c.buildSourceResponse(ctx, outName.ThisID, nil) +} + +// Query Source + +func (c *demoClient) Sources(ctx context.Context, filter *model.SourceSpec) ([]*model.Source, error) { + c.m.RLock() + defer c.m.RUnlock() + if filter != nil && filter.ID != nil { + s, err := c.buildSourceResponse(ctx, *filter.ID, filter) + if err != nil { + if errors.Is(err, errNotFound) { + // not found + return nil, nil + } + return nil, err + } + return []*model.Source{s}, nil + } + + out := []*model.Source{} + if filter != nil && filter.Type != nil { + inType := &srcType{ + Type: *filter.Type, + } + srcTypeNode, err := byKeykv[*srcType](ctx, srcTypeCol, inType.Key(), c) + if err == nil { + sNamespaces := c.buildSourceNamespace(ctx, srcTypeNode, filter) + if len(sNamespaces) > 0 { + out = append(out, &model.Source{ + ID: srcTypeNode.ThisID, + Type: srcTypeNode.Type, + Namespaces: sNamespaces, + }) + } + } + } else { + typeKeys, err := c.kv.Keys(ctx, srcTypeCol) + if err != nil { + return nil, err + } + for _, tk := range typeKeys { + srcTypeNode, err := byKeykv[*srcType](ctx, srcTypeCol, tk, c) + if err != nil { + return nil, err + } + sNamespaces := c.buildSourceNamespace(ctx, srcTypeNode, filter) + if len(sNamespaces) > 0 { + out = append(out, &model.Source{ + ID: srcTypeNode.ThisID, + Type: srcTypeNode.Type, + Namespaces: sNamespaces, + }) + } + } + } + return out, nil +} + +func (c *demoClient) buildSourceNamespace(ctx context.Context, srcTypeNode *srcType, filter *model.SourceSpec) []*model.SourceNamespace { + sNamespaces := []*model.SourceNamespace{} + if filter != nil && filter.Namespace != nil { + inNS := &srcNamespace{ + Parent: srcTypeNode.ThisID, + Namespace: *filter.Namespace, + } + srcNS, err := byKeykv[*srcNamespace](ctx, srcNSCol, inNS.Key(), c) + if err == nil { + sns := c.buildSourceName(ctx, srcNS, filter) + if len(sns) > 0 { + sNamespaces = append(sNamespaces, &model.SourceNamespace{ + ID: srcNS.ThisID, + Namespace: srcNS.Namespace, + Names: sns, + }) + } + } + } else { + for _, nsID := range srcTypeNode.Namespaces { + srcNS, err := byIDkv[*srcNamespace](ctx, nsID, c) + if err != nil { + continue + } + sns := c.buildSourceName(ctx, srcNS, filter) + if len(sns) > 0 { + sNamespaces = append(sNamespaces, &model.SourceNamespace{ + ID: srcNS.ThisID, + Namespace: srcNS.Namespace, + Names: sns, + }) + } + } + } + return sNamespaces +} + +func (c *demoClient) buildSourceName(ctx context.Context, srcNamespace *srcNamespace, filter *model.SourceSpec) []*model.SourceName { + if filter != nil && + filter.Name != nil && + (filter.Tag != nil || filter.Commit != nil) { + inName := &srcNameNode{ + Parent: srcNamespace.ThisID, + Name: *filter.Name, + Tag: nilToEmpty(filter.Tag), + Commit: nilToEmpty(filter.Commit), + } + srcName, err := byKeykv[*srcNameNode](ctx, srcNameCol, inName.Key(), c) + if err != nil { + return nil + } + m := &model.SourceName{ + ID: srcName.ThisID, + Name: srcName.Name, + } + if srcName.Tag != "" { + m.Tag = &srcName.Tag + } + if srcName.Commit != "" { + m.Commit = &srcName.Commit + } + return []*model.SourceName{m} + } + sns := []*model.SourceName{} + for _, nameID := range srcNamespace.Names { + s, err := byIDkv[*srcNameNode](ctx, nameID, c) + if err != nil { + return nil + } + if filter != nil && noMatch(filter.Name, s.Name) { + continue + } + if filter != nil && noMatch(filter.Tag, s.Tag) { + continue + } + if filter != nil && noMatch(filter.Commit, s.Commit) { + continue + } + m := &model.SourceName{ + ID: s.ThisID, + Name: s.Name, + } + if s.Tag != "" { + m.Tag = &s.Tag + } + if s.Commit != "" { + m.Commit = &s.Commit + } + sns = append(sns, m) + } + return sns +} + +// Builds a model.Source to send as GraphQL response, starting from id. +// The optional filter allows restricting output (on selection operations). +func (c *demoClient) buildSourceResponse(ctx context.Context, id string, filter *model.SourceSpec) (*model.Source, error) { + if filter != nil && filter.ID != nil && *filter.ID != id { + return nil, nil + } + + currentID := id + + snl := []*model.SourceName{} + if nameNode, err := byIDkv[*srcNameNode](ctx, currentID, c); err == nil { + if filter != nil && noMatch(filter.Name, nameNode.Name) { + return nil, nil + } + if filter != nil && noMatch(filter.Tag, nameNode.Tag) { + return nil, nil + } + if filter != nil && noMatch(filter.Commit, nameNode.Commit) { + return nil, nil + } + model := &model.SourceName{ + ID: nameNode.ThisID, + Name: nameNode.Name, + } + if nameNode.Tag != "" { + model.Tag = &nameNode.Tag + } + if nameNode.Commit != "" { + model.Commit = &nameNode.Commit + } + snl = append(snl, model) + currentID = nameNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + snsl := []*model.SourceNamespace{} + if namespaceNode, err := byIDkv[*srcNamespace](ctx, currentID, c); err == nil { + if filter != nil && noMatch(filter.Namespace, namespaceNode.Namespace) { + return nil, nil + } + snsl = append(snsl, &model.SourceNamespace{ + ID: namespaceNode.ThisID, + Namespace: namespaceNode.Namespace, + Names: snl, + }) + currentID = namespaceNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + typeNode, err := byIDkv[*srcType](ctx, currentID, c) + if err != nil { + if errors.Is(err, kv.NotFoundError) || errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("%w: ID does not match expected node type for package namespace", errNotFound) + } else { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + } + if filter != nil && noMatch(filter.Type, typeNode.Type) { + return nil, nil + } + s := model.Source{ + ID: typeNode.ThisID, + Type: typeNode.Type, + Namespaces: snsl, + } + return &s, nil +} + +func (c *demoClient) getSourceNameFromInput(ctx context.Context, input model.SourceInputSpec) (*srcNameNode, error) { + inType := &srcType{ + Type: input.Type, + } + srcT, err := byKeykv[*srcType](ctx, srcTypeCol, inType.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package type \"%s\" not found", input.Type) + } + + inNS := &srcNamespace{ + Parent: srcT.ThisID, + Namespace: input.Namespace, + } + srcNS, err := byKeykv[*srcNamespace](ctx, srcNSCol, inNS.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package namespace \"%s\" not found", input.Namespace) + } + + inName := &srcNameNode{ + Parent: srcNS.ThisID, + Name: input.Name, + Tag: nilToEmpty(input.Tag), + Commit: nilToEmpty(input.Commit), + } + srcN, err := byKeykv[*srcNameNode](ctx, srcNameCol, inName.Key(), c) + if err != nil { + return nil, gqlerror.Errorf("Package name \"%s\" not found", input.Name) + } + + return srcN, nil +} + +func (c *demoClient) exactSource(ctx context.Context, filter *model.SourceSpec) (*srcNameNode, error) { + if filter == nil { + return nil, nil + } + if filter.ID != nil { + if srcN, err := byIDkv[*srcNameNode](ctx, *filter.ID, c); err == nil { + return srcN, nil + } else { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + } + if filter.Type != nil && filter.Namespace != nil && filter.Name != nil && (filter.Tag != nil || filter.Commit != nil) { + inType := &srcType{ + Type: *filter.Type, + } + srcT, err := byKeykv[*srcType](ctx, srcTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + + inNS := &srcNamespace{ + Parent: srcT.ThisID, + Namespace: *filter.Namespace, + } + srcNS, err := byKeykv[*srcNamespace](ctx, srcNSCol, inNS.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + + inName := &srcNameNode{ + Parent: srcNS.ThisID, + Name: *filter.Name, + Tag: nilToEmpty(filter.Tag), + Commit: nilToEmpty(filter.Commit), + } + srcN, err := byKeykv[*srcNameNode](ctx, srcNameCol, inName.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + return srcN, nil + } + return nil, nil +} diff --git a/pkg/assembler/backends/inmem/src_test.go b/pkg/assembler/backends/keyvalue/src_test.go similarity index 93% rename from pkg/assembler/backends/inmem/src_test.go rename to pkg/assembler/backends/keyvalue/src_test.go index 3ace205605..09d3c40795 100644 --- a/pkg/assembler/backends/inmem/src_test.go +++ b/pkg/assembler/backends/keyvalue/src_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" @@ -36,9 +36,8 @@ var s1out = &model.Source{ Namespaces: []*model.SourceNamespace{{ Namespace: "github.com/jeff", Names: []*model.SourceName{{ - Name: "myrepo", - Tag: ptrfrom.String("v1.0"), - Commit: ptrfrom.String(""), + Name: "myrepo", + Tag: ptrfrom.String("v1.0"), }}, }}, } @@ -55,7 +54,6 @@ var s2out = &model.Source{ Namespace: "github.com/bob", Names: []*model.SourceName{{ Name: "bobsrepo", - Tag: ptrfrom.String(""), Commit: ptrfrom.String("5e7c41f"), }}, }}, @@ -79,10 +77,7 @@ func Test_demoClient_IngestSources(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - sources: srcTypeMap{}, - index: indexType{}, - } + c, _ := getBackend(context.Background(), nil) got, err := c.IngestSources(ctx, tt.srcInputs) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestSources() error = %v, wantErr %v", err, tt.wantErr) @@ -149,10 +144,7 @@ func Test_demoClient_Sources(t *testing.T) { }, cmp.Ignore()) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - c := &demoClient{ - sources: srcTypeMap{}, - index: indexType{}, - } + c, _ := getBackend(context.Background(), nil) ingestedPkg, err := c.IngestSource(ctx, *tt.srcInput) if (err != nil) != tt.wantErr { t.Errorf("demoClient.IngestSource() error = %v, wantErr %v", err, tt.wantErr) diff --git a/pkg/assembler/backends/inmem/vulnEqual.go b/pkg/assembler/backends/keyvalue/vulnEqual.go similarity index 60% rename from pkg/assembler/backends/inmem/vulnEqual.go rename to pkg/assembler/backends/keyvalue/vulnEqual.go index 1b1f51480b..71a04a2a58 100644 --- a/pkg/assembler/backends/inmem/vulnEqual.go +++ b/pkg/assembler/backends/keyvalue/vulnEqual.go @@ -13,41 +13,48 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" + "fmt" "slices" - "strconv" + "strings" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) // Internal data: link between equal vulnerabilities (vulnEqual) -type ( - vulnerabilityEqualList []*vulnerabilityEqualLink - vulnerabilityEqualLink struct { - id uint32 - vulnerabilities []uint32 - justification string - origin string - collector string - } -) +type vulnerabilityEqualLink struct { + ThisID string + Vulnerabilities []string + Justification string + Origin string + Collector string +} -func (n *vulnerabilityEqualLink) ID() uint32 { return n.id } +func (n *vulnerabilityEqualLink) ID() string { return n.ThisID } +func (n *vulnerabilityEqualLink) Key() string { + return strings.Join([]string{ + fmt.Sprint(n.Vulnerabilities), + n.Justification, + n.Origin, + n.Collector, + }, ":") +} -func (n *vulnerabilityEqualLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 2) - if len(n.vulnerabilities) > 0 && allowedEdges[model.EdgeVulnEqualVulnerability] { - out = append(out, n.vulnerabilities...) +func (n *vulnerabilityEqualLink) Neighbors(allowedEdges edgeMap) []string { + if allowedEdges[model.EdgeVulnEqualVulnerability] { + return n.Vulnerabilities } - return out + return nil } -func (n *vulnerabilityEqualLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.convVulnEqual(n) +func (n *vulnerabilityEqualLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.convVulnEqual(ctx, n) } // Ingest IngestVulnEqual @@ -70,36 +77,35 @@ func (c *demoClient) IngestVulnEqual(ctx context.Context, vulnerability model.Vu func (c *demoClient) ingestVulnEqual(ctx context.Context, vulnerability model.VulnerabilityInputSpec, otherVulnerability model.VulnerabilityInputSpec, vulnEqual model.VulnEqualInputSpec, readOnly bool) (*model.VulnEqual, error) { funcName := "ingestVulnEqual" + + in := &vulnerabilityEqualLink{ + Justification: vulnEqual.Justification, + Origin: vulnEqual.Origin, + Collector: vulnEqual.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - vIDs := make([]uint32, 0, 2) + vIDs := make([]string, 0, 2) + vs := make([]*vulnIDNode, 0, 2) for _, vi := range []model.VulnerabilityInputSpec{vulnerability, otherVulnerability} { - vid, err := getVulnerabilityIDFromInput(c, vi) + v, err := c.getVulnerabilityFromInput(ctx, vi) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - vIDs = append(vIDs, vid) + vs = append(vs, v) + vIDs = append(vIDs, v.ThisID) } slices.Sort(vIDs) + in.Vulnerabilities = vIDs - vs := make([]*vulnIDNode, 0, 2) - for _, vID := range vIDs { - v, _ := byID[*vulnIDNode](vID, c) - vs = append(vs, v) + out, err := byKeykv[*vulnerabilityEqualLink](ctx, vulnEqCol, in.Key(), c) + if err == nil { + return c.convVulnEqual(ctx, out) } - - for _, id := range vs[0].vulnEqualLinks { - ve, err := byID[*vulnerabilityEqualLink](id, c) - if err != nil { - return nil, gqlerror.Errorf("%v :: %v", funcName, err) - } - if slices.Equal(ve.vulnerabilities, vIDs) && - ve.justification == vulnEqual.Justification && - ve.origin == vulnEqual.Origin && - ve.collector == vulnEqual.Collector { - return c.convVulnEqual(ve) - } + if !errors.Is(err, kv.NotFoundError) { + return nil, err } if readOnly { @@ -109,31 +115,31 @@ func (c *demoClient) ingestVulnEqual(ctx context.Context, vulnerability model.Vu return cp, err } - ve := &vulnerabilityEqualLink{ - id: c.getNextID(), - vulnerabilities: vIDs, - justification: vulnEqual.Justification, - origin: vulnEqual.Origin, - collector: vulnEqual.Collector, + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, vulnEqCol, in); err != nil { + return nil, err } - c.index[ve.id] = ve for _, v := range vs { - v.setVulnEqualLinks(ve.id) + if err := v.setVulnEqualLinks(ctx, in.ThisID, c); err != nil { + return nil, err + } + } + if err := setkv(ctx, vulnEqCol, in, c); err != nil { + return nil, err } - c.vulnerabilityEquals = append(c.vulnerabilityEquals, ve) - return c.convVulnEqual(ve) + return c.convVulnEqual(ctx, in) } -func (c *demoClient) convVulnEqual(in *vulnerabilityEqualLink) (*model.VulnEqual, error) { +func (c *demoClient) convVulnEqual(ctx context.Context, in *vulnerabilityEqualLink) (*model.VulnEqual, error) { out := &model.VulnEqual{ - ID: nodeID(in.id), - Justification: in.justification, - Origin: in.origin, - Collector: in.collector, + ID: in.ThisID, + Justification: in.Justification, + Origin: in.Origin, + Collector: in.Collector, } - for _, id := range in.vulnerabilities { - v, err := c.buildVulnResponse(id, nil) + for _, id := range in.Vulnerabilities { + v, err := c.buildVulnResponse(ctx, id, nil) if err != nil { return nil, err } @@ -148,34 +154,29 @@ func (c *demoClient) VulnEqual(ctx context.Context, filter *model.VulnEqualSpec) c.m.RLock() defer c.m.RUnlock() if filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*vulnerabilityEqualLink](id, c) + link, err := byIDkv[*vulnerabilityEqualLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - ve, err := c.convVulnEqual(link) + ve, err := c.convVulnEqual(ctx, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.VulnEqual{ve}, nil } - var search []uint32 + var search []string foundOne := false for _, v := range filter.Vulnerabilities { if !foundOne { - exactVuln, err := c.exactVulnerability(v) + exactVuln, err := c.exactVulnerability(ctx, v) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactVuln != nil { - search = append(search, exactVuln.vulnEqualLinks...) + search = append(search, exactVuln.VulnEqualLinks...) foundOne = true break } @@ -185,19 +186,26 @@ func (c *demoClient) VulnEqual(ctx context.Context, filter *model.VulnEqualSpec) var out []*model.VulnEqual if foundOne { for _, id := range search { - link, err := byID[*vulnerabilityEqualLink](id, c) + link, err := byIDkv[*vulnerabilityEqualLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addVulnIfMatch(out, filter, link) + out, err = c.addVulnIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.vulnerabilityEquals { - var err error - out, err = c.addVulnIfMatch(out, filter, link) + veKeys, err := c.kv.Keys(ctx, vulnEqCol) + if err != nil { + return nil, err + } + for _, vek := range veKeys { + link, err := byKeykv[*vulnerabilityEqualLink](ctx, vulnEqCol, vek, c) + if err != nil { + return nil, err + } + out, err = c.addVulnIfMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -206,13 +214,13 @@ func (c *demoClient) VulnEqual(ctx context.Context, filter *model.VulnEqualSpec) return out, nil } -func (c *demoClient) addVulnIfMatch(out []*model.VulnEqual, +func (c *demoClient) addVulnIfMatch(ctx context.Context, out []*model.VulnEqual, filter *model.VulnEqualSpec, link *vulnerabilityEqualLink) ( []*model.VulnEqual, error, ) { - if noMatch(filter.Justification, link.justification) || - noMatch(filter.Origin, link.origin) || - noMatch(filter.Collector, link.collector) { + if noMatch(filter.Justification, link.Justification) || + noMatch(filter.Origin, link.Origin) || + noMatch(filter.Collector, link.Collector) { return out, nil } for _, vs := range filter.Vulnerabilities { @@ -220,8 +228,8 @@ func (c *demoClient) addVulnIfMatch(out []*model.VulnEqual, continue } found := false - for _, vid := range link.vulnerabilities { - v, err := c.buildVulnResponse(vid, vs) + for _, vid := range link.Vulnerabilities { + v, err := c.buildVulnResponse(ctx, vid, vs) if err != nil { return nil, err } @@ -233,7 +241,7 @@ func (c *demoClient) addVulnIfMatch(out []*model.VulnEqual, return out, nil } } - ve, err := c.convVulnEqual(link) + ve, err := c.convVulnEqual(ctx, link) if err != nil { return nil, err } diff --git a/pkg/assembler/backends/inmem/vulnEqual_test.go b/pkg/assembler/backends/keyvalue/vulnEqual_test.go similarity index 98% rename from pkg/assembler/backends/inmem/vulnEqual_test.go rename to pkg/assembler/backends/keyvalue/vulnEqual_test.go index 629ed1e8aa..3cbcb7ff40 100644 --- a/pkg/assembler/backends/inmem/vulnEqual_test.go +++ b/pkg/assembler/backends/keyvalue/vulnEqual_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -424,13 +425,6 @@ func TestVulnEqual(t *testing.T) { }, ExpQueryErr: false, }, - { - Name: "Query Bad ID", - Query: &model.VulnEqualSpec{ - ID: ptrfrom.String("-123"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -438,7 +432,8 @@ func TestVulnEqual(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -618,7 +613,8 @@ func TestIngestVulnEquals(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -709,7 +705,8 @@ func TestVulnerabilityEqualNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/inmem/vulnMetadata.go b/pkg/assembler/backends/keyvalue/vulnMetadata.go similarity index 54% rename from pkg/assembler/backends/inmem/vulnMetadata.go rename to pkg/assembler/backends/keyvalue/vulnMetadata.go index bc40c41d59..a4d5ff633c 100644 --- a/pkg/assembler/backends/inmem/vulnMetadata.go +++ b/pkg/assembler/backends/keyvalue/vulnMetadata.go @@ -13,42 +13,52 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem +package keyvalue import ( "context" + "errors" "fmt" "reflect" - "strconv" + "strings" "time" "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" "github.com/vektah/gqlparser/v2/gqlerror" ) -type vulnerabilityMetadataList []*vulnerabilityMetadataLink type vulnerabilityMetadataLink struct { - id uint32 - vulnerabilityID uint32 - scoreType model.VulnerabilityScoreType - scoreValue float64 - timestamp time.Time - origin string - collector string + ThisID string + VulnerabilityID string + ScoreType model.VulnerabilityScoreType + ScoreValue float64 + Timestamp time.Time + Origin string + Collector string } -func (n *vulnerabilityMetadataLink) ID() uint32 { return n.id } +func (n *vulnerabilityMetadataLink) ID() string { return n.ThisID } +func (n *vulnerabilityMetadataLink) Key() string { + return strings.Join([]string{ + n.VulnerabilityID, + string(n.ScoreType), + fmt.Sprint(n.ScoreValue), // TODO check that fmt.Sprint(float64) is stable for small diffs (epsilon) fmt.Sprintf("%.2f", f) + timeKey(n.Timestamp), + n.Origin, + n.Collector, + }, ":") +} -func (n *vulnerabilityMetadataLink) Neighbors(allowedEdges edgeMap) []uint32 { - out := make([]uint32, 0, 1) +func (n *vulnerabilityMetadataLink) Neighbors(allowedEdges edgeMap) []string { if allowedEdges[model.EdgeVulnMetadataVulnerability] { - out = append(out, n.vulnerabilityID) + return []string{n.VulnerabilityID} } - return out + return nil } -func (n *vulnerabilityMetadataLink) BuildModelNode(c *demoClient) (model.Node, error) { - return c.buildVulnerabilityMetadata(n, nil, true) +func (n *vulnerabilityMetadataLink) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildVulnerabilityMetadata(ctx, n, nil, true) } // Ingest VulnerabilityMetadata @@ -70,69 +80,51 @@ func (c *demoClient) IngestVulnerabilityMetadata(ctx context.Context, vulnerabil func (c *demoClient) ingestVulnerabilityMetadata(ctx context.Context, vulnerability model.VulnerabilityInputSpec, vulnerabilityMetadata model.VulnerabilityMetadataInputSpec, readOnly bool) (string, error) { funcName := "IngestVulnerabilityMetadata" + + in := &vulnerabilityMetadataLink{ + Timestamp: vulnerabilityMetadata.Timestamp, + ScoreType: vulnerabilityMetadata.ScoreType, + ScoreValue: (vulnerabilityMetadata.ScoreValue), + Origin: vulnerabilityMetadata.Origin, + Collector: vulnerabilityMetadata.Collector, + } + lock(&c.m, readOnly) defer unlock(&c.m, readOnly) - var vulnerabilityLinks []uint32 - - vulnID, err := getVulnerabilityIDFromInput(c, vulnerability) + foundVulnNode, err := c.getVulnerabilityFromInput(ctx, vulnerability) if err != nil { return "", gqlerror.Errorf("%v :: %s", funcName, err) } - foundVulnNode, err := byID[*vulnIDNode](vulnID, c) - if err != nil { - return "", gqlerror.Errorf("%v :: %s", funcName, err) - } - vulnerabilityLinks = foundVulnNode.vulnMetadataLinks - - searchIDs := vulnerabilityLinks + in.VulnerabilityID = foundVulnNode.ThisID - // Don't insert duplicates - duplicate := false - var collectedVulnMetadataLink *vulnerabilityMetadataLink - for _, id := range searchIDs { - v, err := byID[*vulnerabilityMetadataLink](id, c) - if err != nil { - return "", gqlerror.Errorf("%v :: %s", funcName, err) - } - vulnMatch := false - if vulnID != 0 && vulnID == v.vulnerabilityID { - vulnMatch = true - } - if vulnMatch && vulnerabilityMetadata.Timestamp.Equal(v.timestamp) && vulnerabilityMetadata.ScoreType == v.scoreType && - floatEqual(vulnerabilityMetadata.ScoreValue, v.scoreValue) && - vulnerabilityMetadata.Origin == v.origin && vulnerabilityMetadata.Collector == v.collector { + out, err := byKeykv[*vulnerabilityMetadataLink](ctx, vulnMDCol, in.Key(), c) + if err == nil { + return out.ThisID, nil + } + if !errors.Is(err, kv.NotFoundError) { + return "", err + } - collectedVulnMetadataLink = v - duplicate = true - break - } + if readOnly { + c.m.RUnlock() + cv, err := c.ingestVulnerabilityMetadata(ctx, vulnerability, vulnerabilityMetadata, false) + c.m.RLock() // relock so that defer unlock does not panic + return cv, err } - if !duplicate { - if readOnly { - c.m.RUnlock() - cv, err := c.ingestVulnerabilityMetadata(ctx, vulnerability, vulnerabilityMetadata, false) - c.m.RLock() // relock so that defer unlock does not panic - return cv, err - } - // store the link - collectedVulnMetadataLink = &vulnerabilityMetadataLink{ - id: c.getNextID(), - vulnerabilityID: vulnID, - timestamp: vulnerabilityMetadata.Timestamp, - scoreType: vulnerabilityMetadata.ScoreType, - scoreValue: (vulnerabilityMetadata.ScoreValue), - origin: vulnerabilityMetadata.Origin, - collector: vulnerabilityMetadata.Collector, - } - c.index[collectedVulnMetadataLink.id] = collectedVulnMetadataLink - c.vulnerabilityMetadatas = append(c.vulnerabilityMetadatas, collectedVulnMetadataLink) - // set the backlinks - foundVulnNode.setVulnMetadataLinks(collectedVulnMetadataLink.id) + in.ThisID = c.getNextID() + if err := c.addToIndex(ctx, vulnMDCol, in); err != nil { + return "", err + } + if err := foundVulnNode.setVulnMetadataLinks(ctx, in.ThisID, c); err != nil { + return "", err + } + if err := setkv(ctx, vulnMDCol, in, c); err != nil { + return "", err } - return nodeID(collectedVulnMetadataLink.id), nil + return in.ThisID, nil } // Query VulnerabilityMetadata @@ -142,34 +134,29 @@ func (c *demoClient) VulnerabilityMetadata(ctx context.Context, filter *model.Vu funcName := "VulnerabilityMetadata" if filter != nil && filter.ID != nil { - id64, err := strconv.ParseUint(*filter.ID, 10, 32) - if err != nil { - return nil, gqlerror.Errorf("%v :: invalid ID %s", funcName, err) - } - id := uint32(id64) - link, err := byID[*vulnerabilityMetadataLink](id, c) + link, err := byIDkv[*vulnerabilityMetadataLink](ctx, *filter.ID, c) if err != nil { // Not found return nil, nil } // If found by id, ignore rest of fields in spec and return as a match - foundVulnMetadata, err := c.buildVulnerabilityMetadata(link, filter, true) + foundVulnMetadata, err := c.buildVulnerabilityMetadata(ctx, link, filter, true) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } return []*model.VulnerabilityMetadata{foundVulnMetadata}, nil } - var search []uint32 + var search []string foundOne := false if !foundOne && filter != nil && filter.Vulnerability != nil { - exactVuln, err := c.exactVulnerability(filter.Vulnerability) + exactVuln, err := c.exactVulnerability(ctx, filter.Vulnerability) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } if exactVuln != nil { - search = append(search, exactVuln.vulnMetadataLinks...) + search = append(search, exactVuln.VulnMetadataLinks...) foundOne = true } } @@ -177,19 +164,26 @@ func (c *demoClient) VulnerabilityMetadata(ctx context.Context, filter *model.Vu var out []*model.VulnerabilityMetadata if foundOne { for _, id := range search { - link, err := byID[*vulnerabilityMetadataLink](id, c) + link, err := byIDkv[*vulnerabilityMetadataLink](ctx, id, c) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } - out, err = c.addVulnMetadataMatch(out, filter, link) + out, err = c.addVulnMetadataMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } } } else { - for _, link := range c.vulnerabilityMetadatas { - var err error - out, err = c.addVulnMetadataMatch(out, filter, link) + vmdKeys, err := c.kv.Keys(ctx, vulnMDCol) + if err != nil { + return nil, err + } + for _, vmdk := range vmdKeys { + link, err := byKeykv[*vulnerabilityMetadataLink](ctx, vulnMDCol, vmdk, c) + if err != nil { + return nil, err + } + out, err = c.addVulnMetadataMatch(ctx, out, filter, link) if err != nil { return nil, gqlerror.Errorf("%v :: %v", funcName, err) } @@ -199,11 +193,11 @@ func (c *demoClient) VulnerabilityMetadata(ctx context.Context, filter *model.Vu return out, nil } -func (c *demoClient) addVulnMetadataMatch(out []*model.VulnerabilityMetadata, +func (c *demoClient) addVulnMetadataMatch(ctx context.Context, out []*model.VulnerabilityMetadata, filter *model.VulnerabilityMetadataSpec, link *vulnerabilityMetadataLink) ([]*model.VulnerabilityMetadata, error) { - if filter != nil && filter.Timestamp != nil && !filter.Timestamp.Equal(link.timestamp) { + if filter != nil && filter.Timestamp != nil && !filter.Timestamp.Equal(link.Timestamp) { return out, nil } if filter != nil && filter.Comparator != nil { @@ -212,34 +206,34 @@ func (c *demoClient) addVulnMetadataMatch(out []*model.VulnerabilityMetadata, } switch *filter.Comparator { case model.ComparatorEqual: - if link.scoreValue != *filter.ScoreValue { + if link.ScoreValue != *filter.ScoreValue { return out, nil } case model.ComparatorGreater, model.ComparatorGreaterEqual: - if link.scoreValue < *filter.ScoreValue { + if link.ScoreValue < *filter.ScoreValue { return out, nil } case model.ComparatorLess, model.ComparatorLessEqual: - if link.scoreValue > *filter.ScoreValue { + if link.ScoreValue > *filter.ScoreValue { return out, nil } } } else { - if filter != nil && noMatchFloat(filter.ScoreValue, link.scoreValue) { + if filter != nil && noMatchFloat(filter.ScoreValue, link.ScoreValue) { return out, nil } } - if filter != nil && filter.ScoreType != nil && *filter.ScoreType != link.scoreType { + if filter != nil && filter.ScoreType != nil && *filter.ScoreType != link.ScoreType { return out, nil } - if filter != nil && noMatch(filter.Collector, link.collector) { + if filter != nil && noMatch(filter.Collector, link.Collector) { return out, nil } - if filter != nil && noMatch(filter.Origin, link.origin) { + if filter != nil && noMatch(filter.Origin, link.Origin) { return out, nil } - foundVulnMetadata, err := c.buildVulnerabilityMetadata(link, filter, false) + foundVulnMetadata, err := c.buildVulnerabilityMetadata(ctx, link, filter, false) if err != nil { return nil, fmt.Errorf("failed to build vuln metadata node from link") } @@ -249,13 +243,13 @@ func (c *demoClient) addVulnMetadataMatch(out []*model.VulnerabilityMetadata, return append(out, foundVulnMetadata), nil } -func (c *demoClient) buildVulnerabilityMetadata(link *vulnerabilityMetadataLink, filter *model.VulnerabilityMetadataSpec, ingestOrIDProvided bool) (*model.VulnerabilityMetadata, error) { +func (c *demoClient) buildVulnerabilityMetadata(ctx context.Context, link *vulnerabilityMetadataLink, filter *model.VulnerabilityMetadataSpec, ingestOrIDProvided bool) (*model.VulnerabilityMetadata, error) { var vuln *model.Vulnerability var err error if filter != nil && filter.Vulnerability != nil { - if filter.Vulnerability != nil && link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, filter.Vulnerability) + if filter.Vulnerability != nil && link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, filter.Vulnerability) if err != nil { return nil, err } @@ -268,15 +262,15 @@ func (c *demoClient) buildVulnerabilityMetadata(link *vulnerabilityMetadataLink, } } } else { - if link.vulnerabilityID != 0 { - vuln, err = c.buildVulnResponse(link.vulnerabilityID, nil) + if link.VulnerabilityID != "" { + vuln, err = c.buildVulnResponse(ctx, link.VulnerabilityID, nil) if err != nil { return nil, err } } } - if link.vulnerabilityID != 0 { + if link.VulnerabilityID != "" { if vuln == nil && ingestOrIDProvided { return nil, gqlerror.Errorf("failed to retrieve vuln via vulnID") } else if vuln == nil && !ingestOrIDProvided { @@ -285,13 +279,13 @@ func (c *demoClient) buildVulnerabilityMetadata(link *vulnerabilityMetadataLink, } vulnMetadata := &model.VulnerabilityMetadata{ - ID: nodeID(link.id), + ID: link.ThisID, Vulnerability: vuln, - Timestamp: link.timestamp, - ScoreType: model.VulnerabilityScoreType(link.scoreType), - ScoreValue: link.scoreValue, - Origin: link.origin, - Collector: link.collector, + Timestamp: link.Timestamp, + ScoreType: model.VulnerabilityScoreType(link.ScoreType), + ScoreValue: link.ScoreValue, + Origin: link.Origin, + Collector: link.Collector, } return vulnMetadata, nil diff --git a/pkg/assembler/backends/inmem/vulnMetadata_test.go b/pkg/assembler/backends/keyvalue/vulnMetadata_test.go similarity index 98% rename from pkg/assembler/backends/inmem/vulnMetadata_test.go rename to pkg/assembler/backends/keyvalue/vulnMetadata_test.go index 5762c0b4ee..4ed4c63c74 100644 --- a/pkg/assembler/backends/inmem/vulnMetadata_test.go +++ b/pkg/assembler/backends/keyvalue/vulnMetadata_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -24,6 +24,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -789,7 +790,8 @@ func TestIngestVulnMetadata(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1087,7 +1089,8 @@ func TestIngestVulnMetadatas(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -1186,7 +1189,8 @@ func TestVulnMetadataNeighbors(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/backends/keyvalue/vulnerability.go b/pkg/assembler/backends/keyvalue/vulnerability.go new file mode 100644 index 0000000000..e6931efa04 --- /dev/null +++ b/pkg/assembler/backends/keyvalue/vulnerability.go @@ -0,0 +1,415 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package keyvalue + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/vektah/gqlparser/v2/gqlerror" + + "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/pkg/assembler/graphql/model" + "github.com/guacsec/guac/pkg/assembler/kv" +) + +const noVulnType string = "novuln" + +// Internal data: Vulnerability +type vulnTypeStruct struct { + ThisID string + Type string + VulnIDs []string +} +type vulnIDNode struct { + ThisID string + Parent string + VulnID string + CertifyVulnLinks []string + VulnEqualLinks []string + VexLinks []string + VulnMetadataLinks []string +} + +func (n *vulnTypeStruct) ID() string { return n.ThisID } +func (n *vulnIDNode) ID() string { return n.ThisID } + +func (n *vulnTypeStruct) Key() string { + return n.Type +} + +func (n *vulnIDNode) Key() string { + return strings.Join([]string{ + n.Parent, + n.VulnID, + }, ":") +} + +func (n *vulnTypeStruct) Neighbors(allowedEdges edgeMap) []string { + if allowedEdges[model.EdgeVulnerabilityTypeVulnerabilityID] { + return n.VulnIDs + } + return nil +} + +func (n *vulnIDNode) Neighbors(allowedEdges edgeMap) []string { + var out []string + if allowedEdges[model.EdgeVulnerabilityIDVulnerabilityType] { + out = append(out, n.Parent) + } + if allowedEdges[model.EdgeVulnerabilityCertifyVuln] { + out = append(out, n.CertifyVulnLinks...) + } + if allowedEdges[model.EdgeVulnerabilityVulnEqual] { + out = append(out, n.VulnEqualLinks...) + } + if allowedEdges[model.EdgeVulnerabilityCertifyVexStatement] { + out = append(out, n.VexLinks...) + } + if allowedEdges[model.EdgeVulnMetadataVulnerability] { + out = append(out, n.VulnMetadataLinks...) + } + + return out +} + +func (n *vulnTypeStruct) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildVulnResponse(ctx, n.ThisID, nil) +} +func (n *vulnIDNode) BuildModelNode(ctx context.Context, c *demoClient) (model.Node, error) { + return c.buildVulnResponse(ctx, n.ThisID, nil) +} + +// certifyVulnerability back edges +func (n *vulnIDNode) setVulnerabilityLinks(ctx context.Context, id string, c *demoClient) error { + n.CertifyVulnLinks = append(n.CertifyVulnLinks, id) + return setkv(ctx, vulnIDCol, n, c) +} + +// equalVulnerability back edges +func (n *vulnIDNode) setVulnEqualLinks(ctx context.Context, id string, c *demoClient) error { + n.VulnEqualLinks = append(n.VulnEqualLinks, id) + return setkv(ctx, vulnIDCol, n, c) +} + +// certifyVexStatement back edges +func (n *vulnIDNode) setVexLinks(ctx context.Context, id string, c *demoClient) error { + n.VexLinks = append(n.VexLinks, id) + return setkv(ctx, vulnIDCol, n, c) +} + +// vulnerability Metadata back edges +func (n *vulnIDNode) setVulnMetadataLinks(ctx context.Context, id string, c *demoClient) error { + n.VulnMetadataLinks = append(n.VulnMetadataLinks, id) + return setkv(ctx, vulnIDCol, n, c) +} + +func (n *vulnTypeStruct) addVulnID(ctx context.Context, vulnID string, c *demoClient) error { + n.VulnIDs = append(n.VulnIDs, vulnID) + return setkv(ctx, vulnTypeCol, n, c) +} + +// Ingest Vulnerabilities + +func (c *demoClient) IngestVulnerabilities(ctx context.Context, vulns []*model.VulnerabilityInputSpec) ([]*model.Vulnerability, error) { + var modelVulnerabilities []*model.Vulnerability + for _, vuln := range vulns { + modelVuln, err := c.IngestVulnerability(ctx, *vuln) + if err != nil { + return nil, gqlerror.Errorf("IngestVulnerability failed with err: %v", err) + } + modelVulnerabilities = append(modelVulnerabilities, modelVuln) + } + return modelVulnerabilities, nil +} + +func (c *demoClient) IngestVulnerability(ctx context.Context, vuln model.VulnerabilityInputSpec) (*model.Vulnerability, error) { + return c.ingestVuln(ctx, vuln, true) +} + +func (c *demoClient) ingestVuln(ctx context.Context, input model.VulnerabilityInputSpec, readOnly bool) (*model.Vulnerability, error) { + inType := &vulnTypeStruct{ + Type: strings.ToLower(input.Type), + } + c.m.RLock() + outType, err := byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, inType.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outType, err = byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inType.ThisID = c.getNextID() + if err := c.addToIndex(ctx, vulnTypeCol, inType); err != nil { + return nil, err + } + if err := setkv(ctx, vulnTypeCol, inType, c); err != nil { + return nil, err + } + outType = inType + } + c.m.Unlock() + } + + inVulnID := &vulnIDNode{ + Parent: outType.ThisID, + VulnID: strings.ToLower(input.VulnerabilityID), + } + c.m.RLock() + outVulnID, err := byKeykv[*vulnIDNode](ctx, vulnIDCol, inVulnID.Key(), c) + c.m.RUnlock() + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + c.m.Lock() + outVulnID, err = byKeykv[*vulnIDNode](ctx, vulnIDCol, inVulnID.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) { + return nil, err + } + inVulnID.ThisID = c.getNextID() + if err := c.addToIndex(ctx, vulnIDCol, inVulnID); err != nil { + return nil, err + } + if err := setkv(ctx, vulnIDCol, inVulnID, c); err != nil { + return nil, err + } + if err := outType.addVulnID(ctx, inVulnID.ThisID, c); err != nil { + return nil, err + } + outVulnID = inVulnID + } + c.m.Unlock() + } + + // build return GraphQL type + c.m.RLock() + defer c.m.RUnlock() + return c.buildVulnResponse(ctx, outVulnID.ThisID, nil) +} + +// Query Vulnerabilities +func (c *demoClient) Vulnerabilities(ctx context.Context, filter *model.VulnerabilitySpec) ([]*model.Vulnerability, error) { + c.m.RLock() + defer c.m.RUnlock() + if filter != nil && filter.ID != nil { + v, err := c.buildVulnResponse(ctx, *filter.ID, filter) + if err != nil { + if errors.Is(err, errNotFound) { + // not found + return nil, nil + } + return nil, err + } + return []*model.Vulnerability{v}, nil + } + + if filter.NoVuln != nil && !*filter.NoVuln { + if filter.Type != nil && *filter.Type == noVulnType { + return []*model.Vulnerability{}, gqlerror.Errorf("novuln boolean set to false, cannot specify vulnerability type to be novuln") + } + } + + out := []*model.Vulnerability{} + // if novuln is specified, retrieve all "novuln" type nodes + if filter != nil && filter.NoVuln != nil && *filter.NoVuln { + filter.Type = ptrfrom.String(noVulnType) + filter.VulnerabilityID = ptrfrom.String("") + } + + if filter != nil && filter.Type != nil { + inType := &vulnTypeStruct{ + Type: strings.ToLower(*filter.Type), + } + typeStruct, err := byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, inType.Key(), c) + if err == nil { + vulnIDs := c.buildVulnID(ctx, typeStruct, filter) + if len(vulnIDs) > 0 { + out = append(out, &model.Vulnerability{ + ID: typeStruct.ThisID, + Type: typeStruct.Type, + VulnerabilityIDs: vulnIDs, + }) + } + } + } else { + typeKeys, err := c.kv.Keys(ctx, vulnTypeCol) + if err != nil { + return nil, err + } + for _, tk := range typeKeys { + typeStruct, err := byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, tk, c) + if err != nil { + return nil, err + } + vulnIDs := c.buildVulnID(ctx, typeStruct, filter) + if len(vulnIDs) > 0 { + out = append(out, &model.Vulnerability{ + ID: typeStruct.ThisID, + Type: typeStruct.Type, + VulnerabilityIDs: vulnIDs, + }) + } + } + } + return out, nil +} + +func (c *demoClient) buildVulnID(ctx context.Context, typeStruct *vulnTypeStruct, filter *model.VulnerabilitySpec) []*model.VulnerabilityID { + if filter != nil && filter.VulnerabilityID != nil { + inVulnID := &vulnIDNode{ + Parent: typeStruct.ThisID, + VulnID: strings.ToLower(*filter.VulnerabilityID), + } + outVulnID, err := byKeykv[*vulnIDNode](ctx, vulnIDCol, inVulnID.Key(), c) + if err != nil { + return nil + } + return []*model.VulnerabilityID{{ + ID: outVulnID.ThisID, + VulnerabilityID: outVulnID.VulnID, + }} + } + vunIDs := []*model.VulnerabilityID{} + for _, vulnIDID := range typeStruct.VulnIDs { + v, err := byIDkv[*vulnIDNode](ctx, vulnIDID, c) + if err != nil { + return nil + } + if filter != nil && noMatch(toLower(filter.VulnerabilityID), v.VulnID) { + continue + } + vunIDs = append(vunIDs, &model.VulnerabilityID{ + ID: v.ThisID, + VulnerabilityID: v.VulnID, + }) + } + return vunIDs +} + +func (c *demoClient) exactVulnerability(ctx context.Context, filter *model.VulnerabilitySpec) (*vulnIDNode, error) { + if filter == nil { + return nil, nil + } + if filter.ID != nil { + if v, err := byIDkv[*vulnIDNode](ctx, *filter.ID, c); err == nil { + return v, nil + } else { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + } + if filter.Type != nil && filter.VulnerabilityID != nil { + inType := &vulnTypeStruct{ + Type: strings.ToLower(*filter.Type), + } + typeStruct, err := byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, inType.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + + inVulnID := &vulnIDNode{ + Parent: typeStruct.ThisID, + VulnID: strings.ToLower(*filter.VulnerabilityID), + } + vulnID, err := byKeykv[*vulnIDNode](ctx, vulnIDCol, inVulnID.Key(), c) + if err != nil { + if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, err + } + return nil, nil + } + return vulnID, nil + } + return nil, nil +} + +// Builds a model.Vulnerability to send as GraphQL response, starting from id. +// The optional filter allows restricting output (on selection operations). +func (c *demoClient) buildVulnResponse(ctx context.Context, id string, filter *model.VulnerabilitySpec) (*model.Vulnerability, error) { + if filter != nil && filter.ID != nil && *filter.ID != id { + return nil, nil + } + + currentID := id + + var vl []*model.VulnerabilityID + if vulnNode, err := byIDkv[*vulnIDNode](ctx, currentID, c); err == nil { + if filter != nil && noMatch(toLower(filter.VulnerabilityID), vulnNode.VulnID) { + return nil, nil + } + vl = append(vl, &model.VulnerabilityID{ + // IDs are generated as string even though we ask for integers + // See https://github.com/99designs/gqlgen/issues/2561 + ID: vulnNode.ThisID, + VulnerabilityID: vulnNode.VulnID, + }) + currentID = vulnNode.Parent + } else if !errors.Is(err, kv.NotFoundError) && !errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + + typeStruct, err := byIDkv[*vulnTypeStruct](ctx, currentID, c) + if err != nil { + if errors.Is(err, kv.NotFoundError) || errors.Is(err, errTypeNotMatch) { + return nil, fmt.Errorf("%w: ID does not match expected node type for vulnerability", errNotFound) + } else { + return nil, fmt.Errorf("Error retrieving node for id: %v : %w", currentID, err) + } + } + if filter != nil && noMatch(toLower(filter.Type), typeStruct.Type) { + return nil, nil + } + v := model.Vulnerability{ + ID: typeStruct.ThisID, + Type: typeStruct.Type, + VulnerabilityIDs: vl, + } + return &v, nil +} + +func (c *demoClient) getVulnerabilityFromInput(ctx context.Context, input model.VulnerabilityInputSpec) (*vulnIDNode, error) { + inType := &vulnTypeStruct{ + Type: strings.ToLower(input.Type), + } + typeStruct, err := byKeykv[*vulnTypeStruct](ctx, vulnTypeCol, inType.Key(), c) + if err != nil { + return nil, err + } + + inVulnID := &vulnIDNode{ + Parent: typeStruct.ThisID, + VulnID: strings.ToLower(input.VulnerabilityID), + } + vulnID, err := byKeykv[*vulnIDNode](ctx, vulnIDCol, inVulnID.Key(), c) + if err != nil { + return nil, err + } + return vulnID, nil +} diff --git a/pkg/assembler/backends/inmem/vulnerability_test.go b/pkg/assembler/backends/keyvalue/vulnerability_test.go similarity index 96% rename from pkg/assembler/backends/inmem/vulnerability_test.go rename to pkg/assembler/backends/keyvalue/vulnerability_test.go index 35644ca51f..db88a2f977 100644 --- a/pkg/assembler/backends/inmem/vulnerability_test.go +++ b/pkg/assembler/backends/keyvalue/vulnerability_test.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package inmem_test +package keyvalue_test import ( "context" @@ -23,6 +23,7 @@ import ( "github.com/google/go-cmp/cmp" "github.com/guacsec/guac/internal/testing/ptrfrom" + "github.com/guacsec/guac/internal/testing/stablememmap" "github.com/guacsec/guac/pkg/assembler/backends" "github.com/guacsec/guac/pkg/assembler/graphql/model" ) @@ -284,14 +285,6 @@ func TestVulnerability(t *testing.T) { }, Exp: nil, }, - { - Name: "Query invalid ID", - Ingests: []*model.VulnerabilityInputSpec{c1, c2, c3}, - Query: &model.VulnerabilitySpec{ - ID: ptrfrom.String("asdf"), - }, - ExpQueryErr: true, - }, } ignoreID := cmp.FilterPath(func(p cmp.Path) bool { return strings.Compare(".ID", p[len(p)-1].String()) == 0 @@ -299,7 +292,8 @@ func TestVulnerability(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.Name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } @@ -356,7 +350,8 @@ func TestIngestVulnerabilities(t *testing.T) { ctx := context.Background() for _, test := range tests { t.Run(test.name, func(t *testing.T) { - b, err := backends.Get("inmem", nil, nil) + store := stablememmap.GetStore() + b, err := backends.Get("keyvalue", nil, store) if err != nil { t.Fatalf("Could not instantiate testing backend: %v", err) } diff --git a/pkg/assembler/kv/kv.go b/pkg/assembler/kv/kv.go new file mode 100644 index 0000000000..8c6b87e912 --- /dev/null +++ b/pkg/assembler/kv/kv.go @@ -0,0 +1,44 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kv is an interface that the keyvalue backend uses to store data +package kv + +import ( + "context" + "errors" +) + +// Store is an interface to define to serve as a keyvalue store +type Store interface { + + // Retrieve value from store. If not found, returns NotFoundError. Ptr must + // be a pointer to the type of value stored. + Get(ctx context.Context, collection, key string, ptr any) error + + // Sets a value, creates collection if necessary + Set(ctx context.Context, collection, key string, value any) error + + // Returns a slice of all keys for a collection. If collection does not + // exist, return a nil slice. + Keys(ctx context.Context, collection string) ([]string, error) +} + +// Error to return (wrap) on Get if value not found +var NotFoundError = errors.New("Not found") + +// Error to return (wrap) on Get if Ptr is not a pointer, or not the right +// type. +var BadPtrError = errors.New("Bad pointer") diff --git a/pkg/assembler/kv/memmap/memmap.go b/pkg/assembler/kv/memmap/memmap.go new file mode 100644 index 0000000000..9a1677be5f --- /dev/null +++ b/pkg/assembler/kv/memmap/memmap.go @@ -0,0 +1,82 @@ +// +// Copyright 2023 The GUAC Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memmap + +import ( + "context" + "fmt" + "reflect" + + "github.com/guacsec/guac/pkg/assembler/kv" + "golang.org/x/exp/maps" +) + +type Store struct { + m map[string]map[string]any +} + +func GetStore() kv.Store { + return &Store{ + m: make(map[string]map[string]any), + } +} + +func (s *Store) Get(_ context.Context, c, k string, v any) error { + col, ok := s.m[c] + if !ok { + return fmt.Errorf("%w : Collection %q", kv.NotFoundError, c) + } + val, ok := col[k] + if !ok { + return fmt.Errorf("%w : Key %q", kv.NotFoundError, k) + } + + return copyAny(val, v) +} + +func (s *Store) Set(_ context.Context, c, k string, v any) error { + if s.m[c] == nil { + s.m[c] = make(map[string]any) + } + s.m[c][k] = v + return nil +} + +func (s *Store) Keys(_ context.Context, c string) ([]string, error) { + if s.m[c] == nil { + return nil, nil + } + return maps.Keys(s.m[c]), nil +} + +func copyAny(src any, dst any) error { + dP := reflect.ValueOf(dst) + if dP.Kind() != reflect.Pointer { + return fmt.Errorf("%w : Not a pointer", kv.BadPtrError) + } + d := dP.Elem() + if !d.CanSet() { + return fmt.Errorf("%w : Pointer not settable", kv.BadPtrError) + } + s := reflect.ValueOf(src) + // Sometimes dst is an interface containing the same type as src. + // if s.Type() != d.Type() { + // return fmt.Errorf("%w : Source and Destination not same type: %v, %v", + // kv.BadPtrError, s.Type(), d.Type()) + // } + d.Set(s) + return nil +} diff --git a/pkg/cli/store.go b/pkg/cli/store.go index ae07b1fc28..759891f12c 100644 --- a/pkg/cli/store.go +++ b/pkg/cli/store.go @@ -45,7 +45,7 @@ func init() { set.String("csub-tls-cert-file", "", "path to the TLS certificate in PEM format for collect-sub service") set.String("csub-tls-key-file", "", "path to the TLS key in PEM format for collect-sub service") - set.String("gql-backend", "inmem", "backend used for graphql api server: [inmem | arango (experimental) | ent (experimental) | neo4j (unmaintained)]") + set.String("gql-backend", "keyvalue", "backend used for graphql api server: [keyvalue | arango (experimental) | ent (experimental) | neo4j (unmaintained)]") set.Int("gql-listen-port", 8080, "port used for graphql api server") set.String("gql-tls-cert-file", "", "path to the TLS certificate in PEM format for graphql api server") set.String("gql-tls-key-file", "", "path to the TLS key in PEM format for graphql api server") diff --git a/pkg/guacanalytics/patchPlanning_test.go b/pkg/guacanalytics/patchPlanning_test.go index 533225b317..70f08bab4f 100644 --- a/pkg/guacanalytics/patchPlanning_test.go +++ b/pkg/guacanalytics/patchPlanning_test.go @@ -29,7 +29,7 @@ import ( "github.com/guacsec/guac/internal/testing/ptrfrom" "github.com/guacsec/guac/pkg/assembler" "github.com/guacsec/guac/pkg/assembler/backends" - _ "github.com/guacsec/guac/pkg/assembler/backends/inmem" + _ "github.com/guacsec/guac/pkg/assembler/backends/keyvalue" model "github.com/guacsec/guac/pkg/assembler/clients/generated" "github.com/guacsec/guac/pkg/assembler/graphql/generated" "github.com/guacsec/guac/pkg/assembler/graphql/resolvers" @@ -1702,9 +1702,9 @@ func startTestServer() (*http.Server, error) { func getGraphqlTestServer() (*handler.Server, error) { var topResolver resolvers.Resolver - backend, err := backends.Get("inmem", nil, nil) + backend, err := backends.Get("keyvalue", nil, nil) if err != nil { - return nil, fmt.Errorf("error creating inmem backend: %w", err) + return nil, fmt.Errorf("error creating keyvalue backend: %w", err) } topResolver = resolvers.Resolver{Backend: backend}