buildinfo and artifacts (#93)

* pack exports
* basic store implementation
*artifact tests
* reestablish use of input hash instead of taskname
* rewrite hashes to buildinfo
* assure artifact rebuild works as intended

Author: Equanox

architecture.md

@@ -0,0 +1,43 @@
+# bob architecture
+Fine grained infos on bob's features
+
+
+## Buildinfo & Artifact Store
+Bob uses buildinfos and artifacts store to determine if a rebuild is required or if a target could be loaded from the artifact store.
+
+* check if input hash exists in buildinfo store
+* check validity of tragets
+* check if tragets can be loaded from artifact store
+* rebuild
+* write buildinfo
+* write artifacts
+
+```
+@startuml
+ditaa(scale=0.9)
+
+                     /--------------------------------\     
+                     |cFFF                            |     
+                     | `bob build mytask`             |       
+                     \----+----------------------+----/     
+                          |                      |          
+                          |                      |          
+                          |                      |          
+   /----+-----------------+---------+       /----+---------------------------+
+   |cFFF                            |       |cFFF                            |
+   | buildinfo store                |       | artifact store                 | 
+   \--------------------------------/       \--------------------------------/     
+                                          
+@enduml
+```
+
+#### Buildinfo
+A buildinfo item holds general information about the task triggered that build. It also contains a map of input:target combinations for each target in the build chain (child tragets included).
+This allows to validate the integraty and existent of all targets.
+
+It can be accessed by the input hash of a task.
+
+#### Artifact
+A artifact stores the targets and the path information of a task. 
+
+Artifacts can be accesed by the input hash of a task.

bob/aggregate.go

@@ -47,7 +47,9 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 		for variable, value := range boblet.Variables {
 			for key, task := range boblet.Tasks {
 				// TODO: Create and use envvar sanitizer
+
 				task.AddEnvironment(strings.ToUpper(variable), value)
+
 				boblet.Tasks[key] = task
 			}
 		}
@@ -117,6 +119,8 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 		}
 	}
 
+	// TODO: Gather missing tasks from remote  & Unpack?
+
 	// Gather environment from dependent tasks.
 	//
 	// Each export is translated into environment variables named:
@@ -125,6 +129,10 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 	//
 	// The file is prefixed with all paths to make it relative to dir of the the top Bobfile:
 	//   `openapi.yaml => sencond-level/openapi.yaml`
+	//
+	// TODO: Exports should be part of a packed file and should be evaluated when running a playbook or at least after Unpack().
+	// Looks like this is the wrong place to presume that all child tasks are comming from child bobfiles
+	// must exist.
 	for i, task := range aggregate.Tasks {
 
 		for _, dependentTaskName := range task.DependsOn {
@@ -157,6 +165,13 @@ func (b *B) Aggregate() (aggregate *bobfile.Bobfile, err error) {
 		}
 	}
 
+	// Assure tasks are correctly initialised.
+	for i, task := range aggregate.Tasks {
+		task.WithLocalstore(b.local)
+		task.WithBuildinfoStore(b.buildInfoStore)
+		aggregate.Tasks[i] = task
+	}
+
 	return aggregate, aggregate.Verify()
 }
 

bob/bob.go

@@ -8,7 +8,9 @@ import (
 
 	"github.com/Benchkram/errz"
 
+	"github.com/Benchkram/bob/pkg/buildinfostore"
 	"github.com/Benchkram/bob/pkg/file"
+	"github.com/Benchkram/bob/pkg/store"
 )
 
 type B struct {
@@ -22,28 +24,55 @@ type B struct {
 	// dir is bob's working directory.
 	dir string
 
+	// local the place to store artifacts localy
+	local store.Store
+	// TODO: add a remote store
+	// remotestore cas.Store
+
+	buildInfoStore buildinfostore.Store
+
 	// readConfig some commands need a fully initialised bob.
 	// When this is true a `.bob/config` file must exist,
 	// usually done by calling `bob init`
 	readConfig bool
 }
 
-func new() *B {
+func new(opts ...Option) *B {
 	wd, err := os.Getwd()
 	if err != nil {
 		errz.Fatal(err)
 	}
 
-	c := &B{
+	b := &B{
 		DefaultCloneSchema: SSH,
+		dir:                wd,
+	}
 
-		dir: wd,
+	for _, opt := range opts {
+		if opt == nil {
+			continue
+		}
+		opt(b)
 	}
-	return c
+
+	return b
 }
 
-func Bob(opts ...Option) (*B, error) {
-	bob := new()
+// BobWithBaseStoreDir initialises stores in the given directory
+func BobWithBaseStoreDir(baseStoreDir string, opts ...Option) (*B, error) {
+	bob := new(opts...)
+
+	fs, err := Filestore(baseStoreDir)
+	if err != nil {
+		return nil, err
+	}
+	bob.local = fs
+
+	bis, err := BuildinfoStore(baseStoreDir)
+	if err != nil {
+		return nil, err
+	}
+	bob.buildInfoStore = bis
 
 	for _, opt := range opts {
 		if opt == nil {
@@ -52,13 +81,35 @@ func Bob(opts ...Option) (*B, error) {
 		opt(bob)
 	}
 
+	return bob, nil
+}
+
+func Bob(opts ...Option) (*B, error) {
+	bob := new(opts...)
+
 	if bob.readConfig {
 		err := bob.read()
 		if err != nil {
 			return nil, err
 		}
 	}
 
+	if bob.local == nil {
+		fs, err := DefaultFilestore()
+		if err != nil {
+			return nil, err
+		}
+		bob.local = fs
+	}
+
+	if bob.buildInfoStore == nil {
+		bis, err := DefaultBuildinfoStore()
+		if err != nil {
+			return nil, err
+		}
+		bob.buildInfoStore = bis
+	}
+
 	return bob, nil
 }
 

bob/bob_stores.go

@@ -0,0 +1,68 @@
+package bob
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/Benchkram/bob/bob/global"
+	"github.com/Benchkram/bob/pkg/buildinfostore"
+	"github.com/Benchkram/bob/pkg/store"
+	"github.com/Benchkram/bob/pkg/store/filestore"
+	"github.com/Benchkram/errz"
+)
+
+func DefaultFilestore() (s store.Store, err error) {
+	defer errz.Recover(&err)
+
+	home, err := os.UserHomeDir()
+	errz.Fatal(err)
+
+	storeDir := filepath.Join(home, global.BobCacheArtifactsDir)
+	err = os.MkdirAll(storeDir, 0775)
+	errz.Fatal(err)
+
+	return filestore.New(storeDir), nil
+}
+
+func Filestore(dir string) (s store.Store, err error) {
+	defer errz.Recover(&err)
+
+	storeDir := filepath.Join(dir, global.BobCacheArtifactsDir)
+	err = os.MkdirAll(storeDir, 0775)
+	errz.Fatal(err)
+
+	return filestore.New(storeDir), nil
+}
+
+func MustDefaultFilestore() store.Store {
+	s, _ := DefaultFilestore()
+	return s
+}
+
+func DefaultBuildinfoStore() (s buildinfostore.Store, err error) {
+	defer errz.Recover(&err)
+
+	home, err := os.UserHomeDir()
+	errz.Fatal(err)
+
+	storeDir := filepath.Join(home, global.BobCacheBuildinfoDir)
+	err = os.MkdirAll(storeDir, 0775)
+	errz.Fatal(err)
+
+	return buildinfostore.New(storeDir), nil
+}
+
+func BuildinfoStore(dir string) (s buildinfostore.Store, err error) {
+	defer errz.Recover(&err)
+
+	storeDir := filepath.Join(dir, global.BobCacheBuildinfoDir)
+	err = os.MkdirAll(storeDir, 0775)
+	errz.Fatal(err)
+
+	return buildinfostore.New(storeDir), nil
+}
+
+func MustDefaultBuildinfoStore() buildinfostore.Store {
+	s, _ := DefaultBuildinfoStore()
+	return s
+}

bob/bobfile/bobfile.go

@@ -84,13 +84,9 @@ func bobfileRead(dir string) (_ *Bobfile, err error) {
 
 		// Make sure a task is correctly initialised.
 		// TODO: All unitialised must be initialised or get default values.
-		// This mean switching to pointer types for most members.
+		// This means switching to pointer types for most members.
 		task.SetEnv([]string{})
 
-		// if task.TargetDirty.Type == "" {
-		// 	task.T
-		// }
-
 		bobfile.Tasks[key] = task
 	}
 

bob/bobfile/playbook.go

@@ -6,7 +6,10 @@ import (
 )
 
 func (b *Bobfile) Playbook(taskname string) (*playbook.Playbook, error) {
-	pb := playbook.New(taskname)
+	pb := playbook.New(
+		taskname,
+	)
+
 	err := b.Tasks.Walk(taskname, "", func(tn string, task bobtask.Task, err error) error {
 		if err != nil {
 			return err

bob/build.go

@@ -11,6 +11,7 @@ var (
 	ErrNoRebuildRequired = errors.New("no rebuild required")
 )
 
+// Build a task and it's dependecies.
 func (b *B) Build(ctx context.Context, taskname string) (err error) {
 	defer errz.Recover(&err)
 

bob/builder.go

@@ -11,7 +11,7 @@ var _ ctl.Builder = (*builder)(nil)
 
 type BuildFunc func(_ context.Context, runname string, aggregate *bobfile.Bobfile) error
 
-// builder holds all dependecys to build a build task
+// builder holds all dependencys to build a build task
 type builder struct {
 	task      string
 	aggregate *bobfile.Bobfile

bob/clean.go

@@ -0,0 +1,9 @@
+package bob
+
+func (b B) Clean() error {
+	return b.cleanBuildInfoStore()
+}
+
+func (b B) cleanBuildInfoStore() error {
+	return b.buildInfoStore.Clean()
+}

bob/global/global.go

@@ -3,14 +3,17 @@ package global
 import "path/filepath"
 
 const (
-	BobCacheDir      = ".bobcache"
 	BobFileName      = "bob.yaml"
 	BuildToolDir     = ".bob"
 	ConfigFile       = "config"
 	DefaultBuildTask = "build"
 )
 
+// Cache directory
+const BobCacheDir = ".bobcache"
+
 var (
-	FileHashesFileName = filepath.Join(BobCacheDir, "filehashes")
-	TaskHashesFileName = filepath.Join(BobCacheDir, "hashes")
+	BobCacheBuildinfoDir       = filepath.Join(BobCacheDir, "buildinfos")
+	BobCacheTaskHashesFileName = filepath.Join(BobCacheDir, "hashes")
+	BobCacheArtifactsDir       = filepath.Join(BobCacheDir, "artifacts")
 )

bob/options.go

@@ -1,5 +1,10 @@
 package bob
 
+import (
+	"github.com/Benchkram/bob/pkg/buildinfostore"
+	"github.com/Benchkram/bob/pkg/store"
+)
+
 type Option func(b *B)
 
 func WithDir(dir string) Option {
@@ -15,3 +20,15 @@ func WithRequireBobConfig() Option {
 		b.readConfig = true
 	}
 }
+
+func WithFilestore(store store.Store) Option {
+	return func(b *B) {
+		b.local = store
+	}
+}
+
+func WithBuildinfoStore(store buildinfostore.Store) Option {
+	return func(b *B) {
+		b.buildInfoStore = store
+	}
+}