feat: adds types, unit tests, and loading functionality for RegistryC…

…onfig

Closes ortelius#96

cmd/client/commands/build_collection.go

@@ -86,6 +86,10 @@ func (o *BuildCollectionOptions) Validate() error {
 }
 
 func (o *BuildCollectionOptions) Run(ctx context.Context) error {
+	if err := o.Remote.LoadRegistryConfig(); err != nil {
+		return err
+	}
+
 	space, err := workspace.NewLocalWorkspace(o.RootDir)
 	if err != nil {
 		return err
@@ -104,6 +108,7 @@ func (o *BuildCollectionOptions) Run(ctx context.Context) error {
 		orasclient.SkipTLSVerify(o.Insecure),
 		orasclient.WithAuthConfigs(o.Configs),
 		orasclient.WithPlainHTTP(o.PlainHTTP),
+		orasclient.WithRegistryConfig(o.RegistryConfig),
 	)
 	if err != nil {
 		return fmt.Errorf("error configuring client: %v", err)

cmd/client/commands/options/common.go

@@ -32,7 +32,7 @@ func (o *Common) BindFlags(fs *pflag.FlagSet) {
 		"Log level (debug, info, warn, error, fatal)")
 }
 
-// Init initializes default values for Common options.
+// Init initializes default values for Common options at runtime.
 func (o *Common) Init() error {
 	logger, err := log.NewLogger(o.IOStreams.Out, o.LogLevel)
 	if err != nil {

cmd/client/commands/options/remote.go

@@ -1,11 +1,20 @@
 package options
 
-import "github.com/spf13/pflag"
+import (
+	"errors"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	"github.com/uor-framework/uor-client-go/registryclient"
+)
 
 // Remote describes remote configuration options that can be set.
 type Remote struct {
-	Insecure  bool
-	PlainHTTP bool
+	Insecure       bool
+	PlainHTTP      bool
+	RegistryConfig registryclient.RegistryConfig
 }
 
 // BindFlags binds options from a flag set to Remote options.
@@ -14,6 +23,28 @@ func (o *Remote) BindFlags(fs *pflag.FlagSet) {
 	fs.BoolVarP(&o.PlainHTTP, "plain-http", "", o.PlainHTTP, "use plain http and not https when contacting registries")
 }
 
+// LoadRegistryConfig loads the registry config from disk.
+func (o *Remote) LoadRegistryConfig() error {
+	viper.SetConfigName("registry-config")
+	viper.SetConfigType("yaml")
+	viper.AddConfigPath(".")
+	viper.AddConfigPath("$HOME/.uor")
+	viper.AddConfigPath(".")
+	err := viper.ReadInConfig()
+	if err != nil {
+		var configNotFound viper.ConfigFileNotFoundError
+		if errors.As(err, &configNotFound) {
+			return nil
+		}
+		return err
+	}
+	option := viper.DecoderConfigOption(func(config *mapstructure.DecoderConfig) {
+		config.TagName = "json"
+	})
+
+	return viper.Unmarshal(&o.RegistryConfig, option)
+}
+
 // RemoteAuth describes remote authentication configuration options that can be set.
 type RemoteAuth struct {
 	Configs []string

cmd/client/commands/pull.go

@@ -105,6 +105,9 @@ func (o *PullOptions) Validate() error {
 }
 
 func (o *PullOptions) Run(ctx context.Context) error {
+	if err := o.Remote.LoadRegistryConfig(); err != nil {
+		return err
+	}
 
 	if !o.NoVerify {
 		o.Logger.Infof("Checking signature of %s", o.Source)
@@ -139,6 +142,7 @@ func (o *PullOptions) Run(ctx context.Context) error {
 		orasclient.WithPlainHTTP(o.PlainHTTP),
 		orasclient.WithCache(cache),
 		orasclient.WithPullableAttributes(matcher),
+		orasclient.WithRegistryConfig(o.RegistryConfig),
 	)
 	if err != nil {
 		return fmt.Errorf("error configuring client: %v", err)

cmd/client/commands/push.go

@@ -76,6 +76,10 @@ func (o *PushOptions) Run(ctx context.Context) error {
 		return err
 	}
 
+	// Not adding a registry configuration when pushing since only
+	// one reference us being handled at a time.
+	// QUESTION(jpower432): Could this create a problem? I think it would
+	// be more unexpected to publish collection to declared mirrors.
 	client, err := orasclient.NewClient(
 		orasclient.SkipTLSVerify(o.Insecure),
 		orasclient.WithAuthConfigs(o.Configs),

cmd/client/commands/serve.go

@@ -71,6 +71,10 @@ func (o *ServeOptions) Validate() error {
 }
 
 func (o *ServeOptions) Run(ctx context.Context) error {
+	if err := o.Remote.LoadRegistryConfig(); err != nil {
+		return err
+	}
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -84,9 +88,10 @@ func (o *ServeOptions) Run(ctx context.Context) error {
 	manager := defaultmanager.New(cache, o.Logger)
 
 	opts := collectionmanager.ServiceOptions{
-		Insecure:  o.Insecure,
-		PlainHTTP: o.PlainHTTP,
-		PullCache: cache,
+		Insecure:       o.Insecure,
+		PlainHTTP:      o.PlainHTTP,
+		PullCache:      cache,
+		RegistryConfig: o.RegistryConfig,
 	}
 	service := collectionmanager.FromManager(manager, opts)
 

go.mod

@@ -25,7 +25,11 @@ require (
 	sigs.k8s.io/yaml v1.3.0
 )
 
-require github.com/sigstore/cosign v1.12.1
+require (
+	github.com/mitchellh/mapstructure v1.5.0
+	github.com/sigstore/cosign v1.12.1
+	github.com/spf13/viper v1.13.0
+)
 
 require (
 	bitbucket.org/creachadair/shell v0.0.7 // indirect
@@ -174,7 +178,6 @@ require (
 	github.com/miekg/pkcs11 v1.1.1 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.0 // indirect
-	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -213,7 +216,6 @@ require (
 	github.com/soheilhy/cmux v0.1.5 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.13.0 // indirect
 	github.com/spiffe/go-spiffe/v2 v2.1.1 // indirect
 	github.com/subosito/gotenv v1.4.1 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect

registryclient/errors.go

@@ -0,0 +1,12 @@
+package registryclient
+
+import "fmt"
+
+// ErrNoAvailableMirrors denotes that all registry mirrors are not accessible.
+type ErrNoAvailableMirrors struct {
+	Registry string
+}
+
+func (e *ErrNoAvailableMirrors) Error() string {
+	return fmt.Sprintf("registry %q: no avaialble mirrors", e.Registry)
+}

registryclient/orasclient/options.go

@@ -24,14 +24,15 @@ type ClientOption func(o *ClientConfig) error
 
 // ClientConfig contains configuration data for the registry client.
 type ClientConfig struct {
-	outputDir  string
-	configs    []string
-	credFn     func(context.Context, string) (auth.Credential, error)
-	plainHTTP  bool
-	insecure   bool
-	cache      content.Store
-	copyOpts   oras.CopyOptions
-	attributes model.Matcher
+	outputDir      string
+	configs        []string
+	credFn         func(context.Context, string) (auth.Credential, error)
+	plainHTTP      bool
+	insecure       bool
+	cache          content.Store
+	copyOpts       oras.CopyOptions
+	attributes     model.Matcher
+	registryConfig registryclient.RegistryConfig
 }
 
 func (c *ClientConfig) apply(options []ClientOption) error {
@@ -91,6 +92,7 @@ func NewClient(options ...ClientOption) (registryclient.Client, error) {
 	client.destroy = destroy
 	client.cache = config.cache
 	client.attributes = config.attributes
+	client.registryConf = config.registryConfig
 
 	// We are not allowing this to be configurable since
 	// oras file stores turn artifacts into descriptors in
@@ -118,6 +120,16 @@ func WithAuthConfigs(configs []string) ClientOption {
 	}
 }
 
+// WithRegistryConfig defines the configuration for specific registry
+// endpoints. If specified, the configuration for a found registry
+// will override WithSkipTLSVerify and WithPlainHTTP.
+func WithRegistryConfig(registryConf registryclient.RegistryConfig) ClientOption {
+	return func(config *ClientConfig) error {
+		config.registryConfig = registryConf
+		return nil
+	}
+}
+
 // SkipTLSVerify disables TLS certificate checking.
 func SkipTLSVerify(insecure bool) ClientOption {
 	return func(config *ClientConfig) error {

registryclient/orasclient/oras.go

@@ -3,9 +3,11 @@ package orasclient
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"path/filepath"
 	"sort"
 	"sync"
@@ -32,6 +34,7 @@ import (
 type orasClient struct {
 	plainHTTP     bool
 	authClient    *auth.Client
+	registryConf  registryclient.RegistryConfig
 	copyOpts      oras.CopyOptions
 	artifactStore *file.Store
 	cache         content.Store
@@ -132,11 +135,11 @@ func (c *orasClient) LoadCollection(ctx context.Context, reference string) (coll
 }
 
 // Pull performs a copy of OCI artifacts to a local location from a remote location.
-func (c *orasClient) Pull(ctx context.Context, ref string, store content.Store) (ocispec.Descriptor, []ocispec.Descriptor, error) {
+func (c *orasClient) Pull(ctx context.Context, reference string, store content.Store) (ocispec.Descriptor, []ocispec.Descriptor, error) {
 	var allDescs []ocispec.Descriptor
 
 	var from oras.Target
-	repo, err := c.setupRepo(ref)
+	repo, ref, err := c.setupRepo(ctx, reference)
 	if err != nil {
 		return ocispec.Descriptor{}, allDescs, fmt.Errorf("could not create registry target: %w", err)
 	}
@@ -231,27 +234,27 @@ func (c *orasClient) Pull(ctx context.Context, ref string, store content.Store)
 }
 
 // Push performs a copy of OCI artifacts to a remote location.
-func (c *orasClient) Push(ctx context.Context, store content.Store, ref string) (ocispec.Descriptor, error) {
-	repo, err := c.setupRepo(ref)
+func (c *orasClient) Push(ctx context.Context, store content.Store, reference string) (ocispec.Descriptor, error) {
+	repo, updatedRef, err := c.setupRepo(ctx, reference)
 	if err != nil {
 		return ocispec.Descriptor{}, fmt.Errorf("could not create registry target: %w", err)
 	}
 
-	return oras.Copy(ctx, store, ref, repo, ref, c.copyOpts)
+	return oras.Copy(ctx, store, updatedRef, repo, updatedRef, c.copyOpts)
 }
 
 // GetManifest returns the manifest the reference resolves to.
 func (c *orasClient) GetManifest(ctx context.Context, reference string) (ocispec.Descriptor, io.ReadCloser, error) {
-	repo, err := c.setupRepo(reference)
+	repo, updatedRef, err := c.setupRepo(ctx, reference)
 	if err != nil {
 		return ocispec.Descriptor{}, nil, fmt.Errorf("could not create registry target: %w", err)
 	}
-	return repo.FetchReference(ctx, reference)
+	return repo.FetchReference(ctx, updatedRef)
 }
 
 // GetContent retrieves the content for a specified descriptor at a specified reference.
 func (c *orasClient) GetContent(ctx context.Context, reference string, desc ocispec.Descriptor) ([]byte, error) {
-	repo, err := c.setupRepo(reference)
+	repo, _, err := c.setupRepo(ctx, reference)
 	if err != nil {
 		return nil, fmt.Errorf("could not create registry target: %w", err)
 	}
@@ -283,14 +286,90 @@ func (c *orasClient) checkFileStore() error {
 }
 
 // setupRepo configures the client to access the remote repository.
-func (c *orasClient) setupRepo(ref string) (*remote.Repository, error) {
-	repo, err := remote.NewRepository(ref)
+func (c *orasClient) setupRepo(ctx context.Context, reference string) (*remote.Repository, string, error) {
+	reg, err := registryclient.FindRegistry(c.registryConf, reference)
 	if err != nil {
-		return nil, fmt.Errorf("could not create registry target: %w", err)
+		return nil, reference, err
+	}
+
+	repo, err := remote.NewRepository(reference)
+	if err != nil {
+		return nil, reference, fmt.Errorf("could not create registry target: %w", err)
+	}
+
+	// If the incoming reference does not match any registry prefixes,
+	// use the client configuration. If there is a match and the registry
+	// has mirrors configured, try each one before attempting to contact the
+	// input reference (the default).
+	switch {
+	case reg == nil:
+		repo.PlainHTTP = c.plainHTTP
+		repo.Client = c.authClient
+		return repo, reference, nil
+	case len(reg.Mirrors) != 0:
+		repo, ref, err := c.pickMirror(ctx, *reg, reference)
+		if err == nil {
+			return repo, ref, nil
+		}
+
+		var merr *registryclient.ErrNoAvailableMirrors
+		if err != nil && !errors.As(err, &merr) {
+			return nil, reference, err
+		}
+
+		fallthrough
+	default:
+		repo.PlainHTTP = reg.PlainHTTP
+		// FIXME(jpower432): This solution could easily
+		// lead to bugs because the authClient many field that are
+		// pointers or reference types. Come up with something different here.
+		copyAuthClient := *c.authClient
+		copyAuthClient.Client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: reg.SkipTLS,
+				},
+			},
+		}
+		repo.Client = &copyAuthClient
+		return repo, reference, nil
 	}
-	repo.PlainHTTP = c.plainHTTP
-	repo.Client = c.authClient
-	return repo, nil
+}
+
+// pickMirror is used if the reference is linked to a registry in the registry configuration that has mirrors. It returns
+// a configured remote.Repository and rewritten reference per the mirror configuration.
+func (c *orasClient) pickMirror(ctx context.Context, reg registryclient.Registry, ref string) (*remote.Repository, string, error) {
+	pullSources, err := reg.PullSourceFromReference(ref)
+	if err != nil {
+		return nil, ref, err
+	}
+	for _, ps := range pullSources {
+		mirror := ps.Endpoint
+		repo, err := remote.NewRepository(ps.Reference)
+		if err != nil {
+			return nil, ref, fmt.Errorf("could not create registry target: %w", err)
+		}
+		repo.PlainHTTP = mirror.PlainHTTP
+		// FIXME(jpower432): This solution could easily
+		// lead to bugs because the authClient many field that are
+		// pointers or reference types. Come up with something different here.
+		copyAuthClient := *c.authClient
+		copyAuthClient.Client = &http.Client{
+			Transport: &http.Transport{
+				TLSClientConfig: &tls.Config{
+					InsecureSkipVerify: reg.SkipTLS,
+				},
+			},
+		}
+		repo.Client = &copyAuthClient
+
+		// FIXME(jpower432): Resolving work if this is used to publish to a mirror.
+		if _, err := repo.Resolve(ctx, ps.Reference); err == nil {
+			return repo, ps.Reference, nil
+		}
+	}
+
+	return nil, ref, &registryclient.ErrNoAvailableMirrors{Registry: reg.Location}
 }
 
 // loadFiles stores files in a file store and creates descriptors representing each file in the store.