clean-up after nix-shell implementation (#256)

* remove default packages. nix-shell already has them
* no need to whitelist env from host to keep. nix-shell does that
* move env intitialization in BuildNixDependencies; ignore buildCommandPath for hash
* fix target syntax
* remove storePaths from Task, Run, execctl.Cmd
* fix lint; remove unused variable
* no need to has system env. it already exists in t.env which is hashed. ex system=x86_64-linux
* fix store path is displayed on bob build
* use custom nix shell expression to build both packages and files
* remove unused funcs and params
* fix bob build shows no output after nix-collect-garbage -d
* fix env not loaded for build tasks of runs
* add back nic cache assertion
* improve nix install dependencies output
* display progress with . on nix build
* remove DependenciesToStorePathMap
* remove unused StorePath
* format with padding based on package name
* tick every 5s
* add . at start to not be empty for very fast builds
* implement build progress struct
* add back XDG_CACHE_HOME for whitelist env

Author: zuzuleinen

bob/bob.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"runtime"
 
-	"github.com/benchkram/bob/bob/global"
 	"github.com/benchkram/bob/pkg/auth"
 	"github.com/benchkram/bob/pkg/dockermobyutil"
 	"github.com/benchkram/bob/pkg/usererror"
@@ -92,7 +91,6 @@ func newBob(opts ...Option) *B {
 		}
 		opt(b)
 	}
-	b.keepWhitelistEnv()
 
 	return b
 }
@@ -221,13 +219,3 @@ func (b *B) read() (err error) {
 
 	return nil
 }
-
-// keepWhitelistEnv will keep whitelisted env variables
-// from local host
-func (b *B) keepWhitelistEnv() {
-	for _, envKey := range global.EnvWhitelist {
-		if value, exists := os.LookupEnv(envKey); exists {
-			b.env = append(b.env, envKey+"="+value)
-		}
-	}
-}

bob/bobfile/bobfile.go

@@ -209,8 +209,6 @@ func initializeDependencies(dir string, taskDependencies []string, bobfile *Bobf
 		})
 	}
 
-	taskDeps = append(taskDeps, nix.DefaultPackages(bobfile.Nixpkgs)...)
-
 	return nix.UniqueDeps(taskDeps)
 }
 

bob/build.go

@@ -37,6 +37,7 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) {
 		playbook.WithLocalStore(b.local),
 		playbook.WithPushEnabled(b.enablePush),
 		playbook.WithPullEnabled(b.enablePull),
+		playbook.WitNixCache(b.Nix().cache),
 	)
 	errz.Fatal(err)
 

bob/global/env.go

@@ -1,4 +1,4 @@
 package global
 
-// EnvWhitelist keep the whitelisted env variables when hermetic mode is on
-var EnvWhitelist = []string{"HOME", "XDG_CACHE_HOME"}
+// EnvWhitelist keep the whitelisted env variables for nix-shell
+var EnvWhitelist = []string{"XDG_CACHE_HOME", "NIX_SSL_CERT_FILE", "SSL_CERT_FILE"}

bob/install.go

@@ -35,7 +35,7 @@ func (b B) Install() (err error) {
 	fmt.Println()
 
 	if len(allDeps) > 0 {
-		_, err := b.nix.BuildDependencies(allDeps)
+		err = b.nix.BuildDependencies(allDeps)
 		if err != nil {
 			return err
 		}

bob/nix_builder.go

@@ -3,6 +3,7 @@ package bob
 import (
 	"fmt"
 
+	"github.com/benchkram/bob/pkg/envutil"
 	"github.com/benchkram/errz"
 
 	"github.com/benchkram/bob/bob/bobfile"
@@ -62,48 +63,38 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 		return usererror.Wrap(fmt.Errorf("nix is not installed on your system. Get it from %s", nix.DownloadURl()))
 	}
 
-	nixDependencies, err := ag.BTasks.CollectNixDependenciesForTasks(buildTasksInPipeline)
-	errz.Fatal(err)
-
-	runTasksDependencies, err := ag.RTasks.CollectNixDependenciesForTasks(runTasksInPipeline)
-	errz.Fatal(err)
-	nixDependencies = append(nixDependencies, runTasksDependencies...)
-
-	depStorePathMapping, err := nix.BuildDependencies(
-		nix.UniqueDeps(append(nix.DefaultPackages(ag.Nixpkgs), nixDependencies...)),
-		n.cache,
-	)
-	errz.Fatal(err)
-
 	// Resolve nix storePaths from dependencies
 	// and rewrite the affected tasks.
 	for _, name := range buildTasksInPipeline {
 		t := ag.BTasks[name]
 
 		// construct used dependencies for this task
-		deps := nix.DefaultPackages(ag.Nixpkgs)
+		var deps []nix.Dependency
 		deps = append(deps, t.Dependencies()...)
 		deps = nix.UniqueDeps(deps)
 
-		storePaths, err := nix.DependenciesToStorePaths(deps, depStorePathMapping)
+		t.SetNixpkgs(ag.Nixpkgs)
+
+		nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs)
 		errz.Fatal(err)
+		t.SetEnv(envutil.Merge(nixShellEnv, t.Env()))
 
-		t.SetStorePaths(storePaths)
 		ag.BTasks[name] = t
 	}
 
 	for _, name := range runTasksInPipeline {
 		t := ag.RTasks[name]
 
 		// construct used dependencies for this task
-		deps := nix.DefaultPackages(ag.Nixpkgs)
+		var deps []nix.Dependency
 		deps = append(deps, t.Dependencies()...)
 		deps = nix.UniqueDeps(deps)
 
-		storePaths, err := nix.DependenciesToStorePaths(deps, depStorePathMapping)
-		errz.Fatal(err)
+		t.SetNixpkgs(ag.Nixpkgs)
 
-		t.SetStorePaths(storePaths)
+		nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs)
+		errz.Fatal(err)
+		t.SetEnv(envutil.Merge(nixShellEnv, t.Env()))
 
 		ag.RTasks[name] = t
 	}
@@ -112,6 +103,11 @@ func (n *NixBuilder) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipel
 }
 
 // BuildDependencies builds the list of all nix deps
-func (n *NixBuilder) BuildDependencies(deps []nix.Dependency) (nix.DependenciesToStorePathMap, error) {
+func (n *NixBuilder) BuildDependencies(deps []nix.Dependency) error {
 	return nix.BuildDependencies(deps, n.cache)
 }
+
+// BuildEnvironment builds the environment with all nix deps
+func (n *NixBuilder) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) {
+	return nix.BuildEnvironment(deps, nixpkgs, n.cache)
+}

bob/playbook/build_internal.go

@@ -119,7 +119,7 @@ func (p *Playbook) build(ctx context.Context, task *bobtask.Task) (err error) {
 	err = task.Clean()
 	errz.Fatal(err)
 
-	err = task.Run(ctx, p.namePad)
+	err = task.Run(ctx, p.namePad, p.nixCache)
 	if err != nil {
 		taskSuccessFul = false
 		taskErr = err

bob/playbook/options.go

@@ -1,6 +1,9 @@
 package playbook
 
-import "github.com/benchkram/bob/pkg/store"
+import (
+	"github.com/benchkram/bob/pkg/nix"
+	"github.com/benchkram/bob/pkg/store"
+)
 
 type Option func(p *Playbook)
 
@@ -45,3 +48,9 @@ func WithLocalStore(s store.Store) Option {
 		p.localStore = s
 	}
 }
+
+func WitNixCache(c *nix.Cache) Option {
+	return func(p *Playbook) {
+		p.nixCache = c
+	}
+}

bob/playbook/playbook.go

@@ -13,6 +13,7 @@ import (
 	"github.com/benchkram/bob/bobtask/buildinfo"
 	"github.com/benchkram/bob/bobtask/hash"
 	"github.com/benchkram/bob/pkg/boberror"
+	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/bob/pkg/store"
 	"github.com/benchkram/bob/pkg/usererror"
 	"github.com/benchkram/errz"
@@ -74,6 +75,9 @@ type Playbook struct {
 
 	// enablePull allows pulling artifacts from remote store
 	enablePull bool
+
+	// nixCache stores the Nix store paths
+	nixCache *nix.Cache
 }
 
 func New(root string, opts ...Option) *Playbook {

bob/playbook/summary.go

@@ -2,10 +2,10 @@ package playbook
 
 import (
 	"fmt"
-	"time"
 
 	"github.com/benchkram/bob/bobtask"
 	"github.com/benchkram/bob/pkg/boblog"
+	"github.com/benchkram/bob/pkg/format"
 	"github.com/logrusorgru/aurora"
 )
 
@@ -15,7 +15,7 @@ func (p *Playbook) summary(processedTasks []*bobtask.Task) {
 	boblog.Log.V(1).Info("")
 	boblog.Log.V(1).Info(aurora.Bold("● ● ● ●").BrightGreen().String())
 
-	t := fmt.Sprintf("Ran %d tasks in %s", len(processedTasks), displayDuration(p.ExecutionTime()))
+	t := fmt.Sprintf("Ran %d tasks in %s", len(processedTasks), format.DisplayDuration(p.ExecutionTime()))
 
 	boblog.Log.V(1).Info(aurora.Bold(t).BrightGreen().String())
 	for _, t := range processedTasks {
@@ -28,21 +28,11 @@ func (p *Playbook) summary(processedTasks []*bobtask.Task) {
 		execTime := ""
 		status := stat.State()
 		if status != StateNoRebuildRequired {
-			execTime = fmt.Sprintf("\t(%s)", displayDuration(stat.ExecutionTime()))
+			execTime = fmt.Sprintf("\t(%s)", format.DisplayDuration(stat.ExecutionTime()))
 		}
 
 		taskName := t.Name()
 		boblog.Log.V(1).Info(fmt.Sprintf("  %-*s\t%s%s", p.namePad, taskName, status.Summary(), execTime))
 	}
 	boblog.Log.V(1).Info("")
 }
-
-func displayDuration(d time.Duration) string {
-	if d.Minutes() > 1 {
-		return fmt.Sprintf("%.1fm", float64(d)/float64(time.Minute))
-	}
-	if d.Seconds() > 1 {
-		return fmt.Sprintf("%.1fs", float64(d)/float64(time.Second))
-	}
-	return fmt.Sprintf("%.1fms", float64(d)/float64(time.Millisecond)+0.1) // add .1ms so that it never returns 0.0ms
-}

bobrun/run.go

@@ -45,9 +45,8 @@ type Run struct {
 	// in the order which they need to be added to PATH
 	dependencies []nix.Dependency
 
-	// storePaths contain /nix/store/* paths
-	// in the order which they need to be added to PATH
-	storePaths []string
+	// URL of nixpkgs used. If empty, will use local <nixpkgs> channel
+	nixpkgs string
 
 	dir string
 
@@ -66,6 +65,10 @@ func (r *Run) SetEnv(env []string) {
 	r.env = env
 }
 
+func (r *Run) SetNixpkgs(nixpkgs string) {
+	r.nixpkgs = nixpkgs
+}
+
 func (r *Run) Env() []string {
 	return r.env
 }
@@ -89,10 +92,6 @@ func (r *Run) SetDependencies(dependencies []nix.Dependency) {
 	r.dependencies = dependencies
 }
 
-func (r *Run) SetStorePaths(storePaths []string) {
-	r.storePaths = storePaths
-}
-
 func (r *Run) UnmarshalYAML(value *yaml.Node) (err error) {
 	defer errz.Recover(&err)
 
@@ -142,7 +141,6 @@ func (r *Run) Command(ctx context.Context) (rc ctl.Command, err error) {
 			r.name,
 			r.Path,
 			execctl.WithEnv(r.Env()),
-			execctl.WithStorePaths(r.storePaths),
 		)
 		errz.Fatal(err)
 	case RunTypeCompose:

bobrun/with_init.go

@@ -10,8 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/benchkram/bob/pkg/envutil"
-	"github.com/benchkram/bob/pkg/nix"
 	"github.com/benchkram/errz"
 	"github.com/logrusorgru/aurora"
 	"mvdan.cc/sh/expand"
@@ -231,11 +229,6 @@ func (rw *WithInit) shexec(ctx context.Context, cmds []string) (err error) {
 		errz.Fatal(err)
 
 		env := rw.run.Env()
-		if len(rw.run.storePaths) > 0 {
-			nixShellEnv, err := nix.BuildEnvironment(rw.run.dependencies)
-			errz.Fatal(err)
-			env = envutil.Merge(nixShellEnv, env)
-		}
 		pr, pw, err := os.Pipe()
 		errz.Fatal(err)
 

bobtask/hash_in.go

@@ -6,11 +6,9 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"runtime"
 	"sort"
 	"strings"
 
-	"github.com/benchkram/bob/bob/global"
 	"github.com/benchkram/bob/pkg/boblog"
 	"github.com/benchkram/bob/pkg/sliceutil"
 	"gopkg.in/yaml.v2"
@@ -67,26 +65,14 @@ func (t *Task) computeInputHash() (taskHash hash.In, err error) {
 	}
 
 	// Hash the environment
-	env := filterOutWhitelistEnv(t.env)
+	env := filterEnvOfIgnores(t.env)
 	sort.Strings(env)
 	environment := strings.Join(env, ",")
 	err = h.AddBytes(bytes.NewBufferString(environment))
 	if err != nil {
 		return taskHash, fmt.Errorf("failed to write description hash: %w", err)
 	}
 
-	// Hash store paths
-	err = h.AddBytes(bytes.NewBufferString(strings.Join(t.storePaths, "")))
-	if err != nil {
-		return taskHash, fmt.Errorf("failed to write store paths hash: %w", err)
-	}
-
-	// Hash system env
-	err = h.AddBytes(bytes.NewBufferString(strings.Join([]string{runtime.GOOS, runtime.GOARCH}, "-")))
-	if err != nil {
-		return taskHash, fmt.Errorf("failed to write system env hash: %w", err)
-	}
-
 	hashIn := hash.In(hex.EncodeToString(h.Sum()))
 
 	// store hash for reuse
@@ -97,11 +83,13 @@ func (t *Task) computeInputHash() (taskHash hash.In, err error) {
 	return hashIn, nil
 }
 
-func filterOutWhitelistEnv(env []string) []string {
+func filterEnvOfIgnores(env []string) []string {
+	ignore := []string{"buildCommandPath"}
+
 	var result []string
 	for _, v := range env {
 		pair := strings.SplitN(v, "=", 2)
-		if sliceutil.Contains(global.EnvWhitelist, pair[0]) {
+		if sliceutil.Contains(ignore, pair[0]) {
 			continue
 		}
 		result = append(result, v)

bobtask/run.go

@@ -19,14 +19,13 @@ import (
 	"github.com/benchkram/errz"
 )
 
-func (t *Task) Run(ctx context.Context, namePad int) (err error) {
+func (t *Task) Run(ctx context.Context, namePad int, nixCache *nix.Cache) (err error) {
 	defer errz.Recover(&err)
 
-	env := t.Env()
-	if len(t.storePaths) > 0 {
-		nixShellEnv, err := nix.BuildEnvironment(t.dependencies)
+	if len(t.Env()) == 0 {
+		nixShellEnv, err := nix.BuildEnvironment(t.dependencies, t.nixpkgs, nixCache)
 		errz.Fatal(err)
-		env = envutil.Merge(nixShellEnv, env)
+		t.SetEnv(envutil.Merge(nixShellEnv, t.env))
 	}
 
 	for _, run := range t.cmds {
@@ -61,7 +60,7 @@ func (t *Task) Run(ctx context.Context, namePad int) (err error) {
 		r, err := interp.New(
 			interp.Params("-e"),
 			interp.Dir(t.dir),
-			interp.Env(expand.ListEnviron(env...)),
+			interp.Env(expand.ListEnviron(t.Env()...)),
 			interp.StdIO(os.Stdin, pw, pw),
 		)
 

bobtask/task.go

@@ -92,9 +92,8 @@ type Task struct {
 	// in the order which they need to be added to PATH
 	dependencies []nix.Dependency
 
-	// storePaths contain /nix/store/* paths
-	// in the order which they need to be added to PATH
-	storePaths []string
+	// URL of nixpkgs used. If empty, will use local <nixpkgs> channel
+	nixpkgs string
 }
 
 type TargetEntry interface{}