Feat/pull by attribute (#31)

* refactor: decouples graph logic from building logic to allow for graphs to be reusable

Signed-off-by: Jennifer Power <[email protected]>

* fix(parser): fixes parser unit test spelling and error checking issues

Signed-off-by: Jennifer Power <[email protected]>

* feat(model): adds model for node interfaces and implemented node types

Signed-off-by: Jennifer Power <[email protected]>

* feat(cli): adds implementation for pull files by attributes

This only supports one collection currently

Signed-off-by: Jennifer Power <[email protected]>

* docs: adds information for the model and node types in a docs

Adds doc.go to collection and workspace packages

Signed-off-by: Jennifer Power <[email protected]>

* docs: removes unimplemented types from model.md

Signed-off-by: Jennifer Power <[email protected]>

* fix: adds fixes for linting errors found by staticcheck

Signed-off-by: Jennifer Power <[email protected]>

* docs: update README.md with pull-by-attibutes steps

Signed-off-by: Jennifer Power <[email protected]>

* fix: fixes formatting in collection.go

Signed-off-by: Jennifer Power <[email protected]>

.gitignore

@@ -6,6 +6,7 @@ coverage.out
 sha256sum.txt
 **/test-output
 client-workspace
+dataset-config.yaml
 
 
 # Logs.

README.md

@@ -10,6 +10,8 @@ embedded in UOR artifacts.
 
 To learn more about Universal Runtime visit the UOR Framework website at https://uor-framework.github.io.
 
+> WARNING: The repository is under active development and the API is subject to change.
+
 ## Development
 
 ### Requirements
@@ -53,11 +55,16 @@ client build my-directory --output my-workspace
 client push my-workspace localhost:5000/myartifacts:latest
 ```
 
-### Pull artifact to a location
+### Pull UOR collection to a location
 ```
 client pull localhost:5000/myartifacts:latest my-output-directory
 ```
 
+### Pull subsets of a UOR collection to a location by attribute
+```
+client pull localhost:5000/myartifacts:latest my-output-directory --attributes key=value
+```
+
 ## Getting Started
 
 1. Create a new directory. 
@@ -104,9 +111,16 @@ client push my-workspace localhost:5000/test/dataset:latest --dsconfig dataset-c
 6. Optionally inspect the OCI manifest of the dataset:
   `curl -H "Accept: application/vnd.oci.image.manifest.v1+json" <servername>:<port>/v2/<namespace>/<repo>/manifests/<digest or tag>`
 
-7. Optionally pull the artifact back down to verify the content with `client pull`:
+7. Optionally pull the collection back down to verify the content with `client pull`:
   `client pull localhost:5000/test/dataset:latest my-output-directory`
 
+8. Optionally pull a subset of the collection back down to verify the content with `client pull`:
+  `client pull localhost:5000/test/dataset:latest my-output-directory --attributes "fiction=true"`
+
+# Glossary
+
+`collection`: a collection of linked files represented as on OCI artifact
+
 
 
 

attributes/attributes.go

@@ -0,0 +1,103 @@
+package attributes
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/uor-framework/client/model"
+)
+
+// Attributes implements the model.Attributes interface
+// using a multi-map storing a set of values.
+// The current implementation would allow for aggregation of the attributes
+// of child nodes to the parent nodes.
+type Attributes map[string]map[string]struct{}
+
+var _ model.Attributes = &Attributes{}
+
+// Find returns all values stored for a specified key.
+func (a Attributes) Find(key string) []string {
+	valSet, exists := a[key]
+	if !exists {
+		return nil
+	}
+	var vals []string
+	for val := range valSet {
+		vals = append(vals, val)
+	}
+	return vals
+}
+
+// Exists returns whether a key,value pair exists in the
+// attribute set.
+func (a Attributes) Exists(key, value string) bool {
+	vals, exists := a[key]
+	if !exists {
+		return false
+	}
+	_, valExists := vals[value]
+	return valExists
+}
+
+// Strings returns a string representation of the
+// attribute set.
+func (a Attributes) String() string {
+	out := new(strings.Builder)
+	keys := make([]string, 0, len(a))
+	for k := range a {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		vals := a.List()[key]
+		sort.Strings(vals)
+		for _, val := range vals {
+			line := fmt.Sprintf("%s=%s,", key, val)
+			out.WriteString(line)
+		}
+	}
+	return strings.TrimSuffix(out.String(), ",")
+}
+
+// List will list all key, value pairs for the attributes in a
+// consumable format.
+func (a Attributes) List() map[string][]string {
+	list := make(map[string][]string, len(a))
+	for key, vals := range a {
+		for val := range vals {
+			list[key] = append(list[key], val)
+		}
+	}
+	return list
+}
+
+// Len returns the length of the attribute set.
+func (a Attributes) Len() int {
+	return len(a)
+}
+
+// Merge will merge the input Attributes with the receiver.
+func (a Attributes) Merge(attr model.Attributes) {
+	for key, vals := range attr.List() {
+		for _, val := range vals {
+			sub := a[key]
+			sub[val] = struct{}{}
+		}
+	}
+}
+
+// AnnotationsToAttributes converts annotations from a descriptors
+// to an Attribute type.
+func AnnotationsToAttributes(annotations map[string]string) model.Attributes {
+	attr := Attributes{}
+	for key, value := range annotations {
+		curr, exists := attr[key]
+		if !exists {
+			curr = map[string]struct{}{}
+		}
+		curr[value] = struct{}{}
+		attr[key] = curr
+	}
+	return attr
+}

attributes/attributes_test.go

@@ -0,0 +1,63 @@
+package attributes
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExists(t *testing.T) {
+	attributes := Attributes{
+		"kind": map[string]struct{}{
+			"jpg": {},
+			"txt": {},
+		},
+		"name": map[string]struct{}{
+			"fish.jpg": {},
+		},
+	}
+	require.True(t, attributes.Exists("kind", "jpg"))
+	require.False(t, attributes.Exists("kind", "png"))
+}
+
+func TestFind(t *testing.T) {
+	attributes := Attributes{
+		"kind": map[string]struct{}{
+			"jpg": {},
+			"txt": {},
+		},
+		"name": map[string]struct{}{
+			"fish.jpg": {},
+		},
+	}
+	result := attributes.Find("kind")
+	require.Len(t, result, 2)
+	require.Contains(t, result, "jpg")
+	require.Contains(t, result, "txt")
+}
+
+func TestAttributes_String(t *testing.T) {
+	expString := `kind=jpg,kind=txt,name=fish.jpg`
+	attributes := Attributes{
+		"kind": map[string]struct{}{
+			"jpg": {},
+			"txt": {},
+		},
+		"name": map[string]struct{}{
+			"fish.jpg": {},
+		},
+	}
+	require.Equal(t, expString, attributes.String())
+}
+
+func TestAnnotationsToAttributes(t *testing.T) {
+	expList := map[string][]string{
+		"kind": {"jpg"},
+		"name": {"fish.jpg"},
+	}
+	annotations := map[string]string{
+		"kind": "jpg",
+		"name": "fish.jpg",
+	}
+	require.Equal(t, expList, AnnotationsToAttributes(annotations).List())
+}

attributes/doc.go

@@ -0,0 +1,17 @@
+/*
+Copyright 2022 UOR-Framework Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package attributes
+
+// This package defines types and methods for performing attributing matching,
+// implementation, and node searching.

attributes/matcher.go

@@ -0,0 +1,77 @@
+package attributes
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/uor-framework/client/model"
+)
+
+var (
+	_ model.Matcher = &PartialAttributeMatcher{}
+	_ model.Matcher = &ExactAttributeMatcher{}
+)
+
+// PartialAttributeMatcher contains configuration data for searching for a node by attribute.
+// This matcher will check that the node attributes
+type PartialAttributeMatcher map[string]string
+
+// String list all attributes in the Matcher in a string format.
+func (m PartialAttributeMatcher) String() string {
+	return renderMatcher(m)
+}
+
+// Matches determines whether a node has all required attributes.
+func (m PartialAttributeMatcher) Matches(n model.Node) bool {
+	attr := n.Attributes()
+	if attr == nil {
+		return false
+	}
+	for key, value := range m {
+		if exist := attr.Exists(key, value); !exist {
+			return false
+		}
+	}
+	return true
+}
+
+// ExactAttributeMatcher contains configuration data for searching for a node by attribute.
+type ExactAttributeMatcher map[string]string
+
+// String list all attributes in the Matcher in a string format.
+func (m ExactAttributeMatcher) String() string {
+	return renderMatcher(m)
+}
+
+// Matches determines whether a node has all required attributes.
+func (m ExactAttributeMatcher) Matches(n model.Node) bool {
+	attr := n.Attributes()
+	if attr == nil {
+		return false
+	}
+	if len(m) != attr.Len() {
+		return false
+	}
+	for key, value := range m {
+		if exist := attr.Exists(key, value); !exist {
+			return false
+		}
+	}
+	return true
+}
+
+// renderMatcher will render an attribute matcher as a string
+func renderMatcher(m map[string]string) string {
+	out := new(strings.Builder)
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		line := fmt.Sprintf("%s=%s,", key, m[key])
+		out.WriteString(line)
+	}
+	return strings.TrimSuffix(out.String(), ",")
+}

attributes/matcher_test.go

@@ -0,0 +1,29 @@
+package attributes
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/uor-framework/client/util/testutils"
+)
+
+func TestPartialMatcher_String(t *testing.T) {
+	expString := `kind=jpg,name=fish.jpg`
+	m := PartialAttributeMatcher{
+		"kind": "jpg",
+		"name": "fish.jpg",
+	}
+	require.Equal(t, expString, m.String())
+}
+
+func TestPartialMatches(t *testing.T) {
+	mockAttributes := testutils.MockAttributes{
+		"kind":    "jpg",
+		"name":    "fish.jpg",
+		"another": "attribute",
+	}
+
+	n := &testutils.MockNode{A: mockAttributes}
+	m := PartialAttributeMatcher{"kind": "jpg"}
+	require.True(t, m.Matches(n))
+}

builder/api/v1alpha1/config_types.go

@@ -1,6 +1,6 @@
 package v1alpha1
 
-// DataSetConfiguration object kind.
+// DataSetConfigurationKind object kind of DataSetConfiguration.
 const DataSetConfigurationKind = "DataSetConfiguration"
 
 // DataSetConfiguration configures a dataset