From 46e9719da473484d6208753f005d2bf29fedc483 Mon Sep 17 00:00:00 2001 From: equanox Date: Wed, 17 Jan 2024 11:56:10 +0100 Subject: [PATCH 1/5] add shell directive --- bob/bobfile/bobfile.go | 6 +++++ bob/build.go | 2 +- bob/nix-builder/nix_builder.go | 26 +++++++++++++++++++++- pkg/nix/nix.go | 40 ++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 2 deletions(-) diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index b6454621..d4e0a000 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -65,8 +65,14 @@ type Bobfile struct { // RTasks run tasks RTasks bobrun.RunMap `yaml:"run"` + // Dependencies are nix packages used on a global scope. + // Mutually exclusive with Shell. ??Overwrites task based dependencies.?? Dependencies []string `yaml:"dependencies"` + // Shell specifies a shell.nix file as usually used by nix-shell. + // This is mutualy exclusive with Dependencies. + Shell string `yaml:"shell"` + // Nixpkgs specifies an optional nixpkgs source. Nixpkgs string `yaml:"nixpkgs"` diff --git a/bob/build.go b/bob/build.go index 54089abd..fffa75d5 100644 --- a/bob/build.go +++ b/bob/build.go @@ -48,7 +48,7 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) { } // AggregateWithNixDeps does aggregation together with evaluating nix dependecies. -// Nic dependencies are altering a tasks input hash. +// Nix dependencies are altering a tasks input hash. // Use this function for building `bob inspect` cmds. func (b *B) AggregateWithNixDeps(taskName string) (aggregate *bobfile.Bobfile, err error) { defer errz.Recover(&err) diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index 1ce21ddc..3f52222b 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -2,8 +2,11 @@ package nixbuilder import ( "fmt" + "os" + "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/envutil" + "github.com/benchkram/bob/pkg/file" "github.com/benchkram/errz" "github.com/benchkram/bob/bob/bobfile" @@ -88,6 +91,28 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run return usererror.Wrap(fmt.Errorf("nix is not installed on your system. Get it from %s", nix.DownloadURl())) } + if ag.Shell != "" { + boblog.Log.Info(fmt.Sprintf("using shell file %s, ignoring dependencies", ag.Shell)) + // hash shell file + if !file.Exists(ag.Shell) { + return usererror.Wrap(fmt.Errorf("shell file %s does not exist", ag.Shell)) + } + + nixShellEnv, err := nix.NixShell(ag.Shell) + errz.Fatal(err) + + f, err := os.ReadFile(ag.Shell) + errz.Fatal(err) + hash := envutil.Hash(string(f)) + n.envStore[envutil.Hash(hash)] = nixShellEnv + for _, name := range buildTasksInPipeline { + t := ag.BTasks[name] + t.SetEnvID(envutil.Hash(hash)) + ag.BTasks[name] = t + } + return nil + } + // Resolve nix storePaths from dependencies // and rewrite the affected tasks. for _, name := range buildTasksInPipeline { @@ -153,7 +178,6 @@ func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string return nix.BuildEnvironment(deps, nixpkgs, n.cache, n.shellCache) } -// Clean removes all cached nix dependencies func (n *NB) Clean() (err error) { return n.cache.Clean() } diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index 34c260e6..6565a852 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -336,3 +336,43 @@ func HashDependencies(deps []Dependency) (_ string, err error) { } return string(h.Sum()), nil } + +// NixShell returns the environment of a nix-shell command +func NixShell(path string) ([]string, error) { + + args := []string{} + for _, envKey := range global.EnvWhitelist { + if _, exists := os.LookupEnv(envKey); exists { + args = append(args, []string{"--keep", envKey}...) + } + } + args = append(args, []string{"--pure", "--command", "env"}...) + args = append(args, path) + cmd := exec.Command("nix-shell", args...) + + var out bytes.Buffer + var errBuf bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errBuf + + env := strings.Split(out.String(), "\n") + err := cmd.Run() + if err != nil { + return nil, prepareRunError(err, cmd.String(), errBuf) + } + + // if NIX_SSL_CERT_FILE && SSL_CERT_FILE are set to /no-cert-file.crt unset them + var clearedEnv []string + for _, e := range env { + pair := strings.SplitN(e, "=", 2) + if pair[0] == "NIX_SSL_CERT_FILE" && pair[1] == "/no-cert-file.crt" { + continue + } + if pair[0] == "SSL_CERT_FILE" && pair[1] == "/no-cert-file.crt" { + continue + } + clearedEnv = append(clearedEnv, e) + } + + return clearedEnv, nil +} From 007ab63816d32a14d1b930c3026737b635bc8424 Mon Sep 17 00:00:00 2001 From: equanox Date: Mon, 22 Jan 2024 17:16:50 +0100 Subject: [PATCH 2/5] pipe nix-shell env through buil;d task --- bob/nix-builder/nix_builder.go | 6 +++--- pkg/nix/nix.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index 3f52222b..710bf592 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -2,11 +2,11 @@ package nixbuilder import ( "fmt" - "os" "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/envutil" "github.com/benchkram/bob/pkg/file" + "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/errz" "github.com/benchkram/bob/bob/bobfile" @@ -101,9 +101,9 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run nixShellEnv, err := nix.NixShell(ag.Shell) errz.Fatal(err) - f, err := os.ReadFile(ag.Shell) + hash, err := filehash.Hash(ag.Shell) errz.Fatal(err) - hash := envutil.Hash(string(f)) + n.envStore[envutil.Hash(hash)] = nixShellEnv for _, name := range buildTasksInPipeline { t := ag.BTasks[name] diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index 6565a852..042859bf 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -349,13 +349,13 @@ func NixShell(path string) ([]string, error) { args = append(args, []string{"--pure", "--command", "env"}...) args = append(args, path) cmd := exec.Command("nix-shell", args...) + boblog.Log.V(5).Info(fmt.Sprintf("Executing command:\n %s", cmd.String())) var out bytes.Buffer var errBuf bytes.Buffer cmd.Stdout = &out cmd.Stderr = &errBuf - env := strings.Split(out.String(), "\n") err := cmd.Run() if err != nil { return nil, prepareRunError(err, cmd.String(), errBuf) @@ -363,7 +363,7 @@ func NixShell(path string) ([]string, error) { // if NIX_SSL_CERT_FILE && SSL_CERT_FILE are set to /no-cert-file.crt unset them var clearedEnv []string - for _, e := range env { + for _, e := range strings.Split(out.String(), "\n") { pair := strings.SplitN(e, "=", 2) if pair[0] == "NIX_SSL_CERT_FILE" && pair[1] == "/no-cert-file.crt" { continue From 173b3bdbf4571b46418828142ffe33285f0af356 Mon Sep 17 00:00:00 2001 From: equanox Date: Tue, 6 Feb 2024 08:33:16 +0100 Subject: [PATCH 3/5] unify buildenv to use shell.nix [wip] --- bob/nix-builder/nix_builder.go | 11 +++++++++-- pkg/nix/nix.go | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index 710bf592..e3b76688 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -110,6 +110,7 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run t.SetEnvID(envutil.Hash(hash)) ag.BTasks[name] = t } + // TODO: run tasks return nil } @@ -139,7 +140,7 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run } // FIXME: environment cache is a workaround... - // either use envSTore and adapt run tasks to use ist as well + // either use envSTore and adapt run tasks to use it as well // or remove run tasks entirely. environmentCache := make(map[string][]string) for _, name := range runTasksInPipeline { @@ -175,9 +176,15 @@ func (n *NB) BuildDependencies(deps []nix.Dependency) error { // BuildEnvironment builds the environment with all nix deps func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) { - return nix.BuildEnvironment(deps, nixpkgs, n.cache, n.shellCache) + return nix.BuildEnvironment(deps, nixpkgs, + nix.BuildEnvironmentArgs{ + Cache: n.cache, + ShellCache: n.shellCache, + }, + ) } +// Clean removes all cached nix dependencies func (n *NB) Clean() (err error) { return n.cache.Clean() } diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index 042859bf..a944607f 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -211,14 +211,43 @@ func source(nixpkgs string) string { return "" } +type BuildEnvironmentArgs struct { + + // Cache is used to store the store path of the nix dependencies + Cache *Cache + // ShellCache is used to store the environment of a nix-shell command + ShellCache *ShellCache + + // Path to a shell.nix file. + ShellDotNix *string + // List of dependencies imported from the + // shell.nix file. Bob doesn't parse the imports + // therefore it's required to pass all dependencies. + NixShellImports []string +} + // BuildEnvironment is running nix-shell for a list of dependencies and fetch its whole environment // // nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' -E nixExpressionFromDeps // // nix shell can be started with empty list of packages so this method works with empty deps as well -func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCache *ShellCache) (_ []string, err error) { +func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { defer errz.Recover(&err) + var cache *Cache + var shellCache *ShellCache + var shellDotNix *string + + if args.Cache != nil { + cache = args.Cache + } + if args.ShellCache != nil { + shellCache = args.ShellCache + } + if args.ShellDotNix != nil { + shellDotNix = args.ShellDotNix + } + // building dependencies with nix-build to display store paths to output err = BuildDependencies(deps, cache) errz.Fatal(err) @@ -233,6 +262,9 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCach } arguments = append(arguments, []string{"--command", "env"}...) arguments = append(arguments, []string{"--expr", expression}...) + if shellDotNix != nil { + arguments = append(arguments, *shellDotNix) + } cmd := exec.Command("nix-shell", "--pure") cmd.Args = append(cmd.Args, arguments...) From c927ab446ba418a7c348f21444774a8d827118f2 Mon Sep 17 00:00:00 2001 From: equanox Date: Wed, 7 Feb 2024 16:52:44 +0100 Subject: [PATCH 4/5] unify build development --- bob/bobfile/bobfile.go | 2 +- bob/nix-builder/nix_builder.go | 68 +++++++++++++++++++--------------- cli/cmd_inspect.go | 9 ++++- pkg/nix/nix.go | 17 +++++++-- 4 files changed, 61 insertions(+), 35 deletions(-) diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index d4e0a000..5cc315f9 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -71,7 +71,7 @@ type Bobfile struct { // Shell specifies a shell.nix file as usually used by nix-shell. // This is mutualy exclusive with Dependencies. - Shell string `yaml:"shell"` + ShellDotNix string `yaml:"shell"` // Nixpkgs specifies an optional nixpkgs source. Nixpkgs string `yaml:"nixpkgs"` diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index e3b76688..87b59cfc 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -3,10 +3,7 @@ package nixbuilder import ( "fmt" - "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/envutil" - "github.com/benchkram/bob/pkg/file" - "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/errz" "github.com/benchkram/bob/bob/bobfile" @@ -91,28 +88,32 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run return usererror.Wrap(fmt.Errorf("nix is not installed on your system. Get it from %s", nix.DownloadURl())) } - if ag.Shell != "" { - boblog.Log.Info(fmt.Sprintf("using shell file %s, ignoring dependencies", ag.Shell)) - // hash shell file - if !file.Exists(ag.Shell) { - return usererror.Wrap(fmt.Errorf("shell file %s does not exist", ag.Shell)) - } - - nixShellEnv, err := nix.NixShell(ag.Shell) - errz.Fatal(err) - - hash, err := filehash.Hash(ag.Shell) - errz.Fatal(err) - - n.envStore[envutil.Hash(hash)] = nixShellEnv - for _, name := range buildTasksInPipeline { - t := ag.BTasks[name] - t.SetEnvID(envutil.Hash(hash)) - ag.BTasks[name] = t - } - // TODO: run tasks - return nil + var shellDotNix *string + if ag.ShellDotNix != "" { + shellDotNix = &ag.ShellDotNix } + // if ag.Shell != "" { + // boblog.Log.Info(fmt.Sprintf("using shell file %s, ignoring dependencies", ag.Shell)) + // // hash shell file + // if !file.Exists(ag.Shell) { + // return usererror.Wrap(fmt.Errorf("shell file %s does not exist", ag.Shell)) + // } + + // nixShellEnv, err := nix.NixShell(ag.Shell) + // errz.Fatal(err) + + // hash, err := filehash.Hash(ag.Shell) + // errz.Fatal(err) + + // n.envStore[envutil.Hash(hash)] = nixShellEnv + // for _, name := range buildTasksInPipeline { + // t := ag.BTasks[name] + // t.SetEnvID(envutil.Hash(hash)) + // ag.BTasks[name] = t + // } + // // TODO: run tasks + // return nil + // } // Resolve nix storePaths from dependencies // and rewrite the affected tasks. @@ -130,7 +131,9 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run errz.Fatal(err) if _, ok := n.envStore[envutil.Hash(hash)]; !ok { - nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs) + nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, + BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + ) errz.Fatal(err) n.envStore[envutil.Hash(hash)] = nixShellEnv } @@ -157,7 +160,9 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run errz.Fatal(err) if _, ok := environmentCache[hash]; !ok { - nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs) + nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, + BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + ) errz.Fatal(err) environmentCache[hash] = nixShellEnv } @@ -174,12 +179,17 @@ func (n *NB) BuildDependencies(deps []nix.Dependency) error { return nix.BuildDependencies(deps, n.cache) } +type BuildEnvironmentArgs struct { + ShellDotNix *string +} + // BuildEnvironment builds the environment with all nix deps -func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) { +func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { return nix.BuildEnvironment(deps, nixpkgs, nix.BuildEnvironmentArgs{ - Cache: n.cache, - ShellCache: n.shellCache, + Cache: n.cache, + ShellCache: n.shellCache, + ShellDotNix: args.ShellDotNix, }, ) } diff --git a/cli/cmd_inspect.go b/cli/cmd_inspect.go index dbfb4099..0a81c620 100644 --- a/cli/cmd_inspect.go +++ b/cli/cmd_inspect.go @@ -8,6 +8,7 @@ import ( "sort" "github.com/benchkram/bob/bob" + nixbuilder "github.com/benchkram/bob/bob/nix-builder" "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/bob/pkg/usererror" @@ -77,7 +78,13 @@ func runEnv(taskname string) { } task = bobfile.BTasks[taskname] - taskEnv, err := b.Nix().BuildEnvironment(task.Dependencies(), task.Nixpkgs()) + var shellDotNix *string + if bobfile.ShellDotNix != "" { + shellDotNix = &bobfile.ShellDotNix + } + taskEnv, err := b.Nix().BuildEnvironment(task.Dependencies(), task.Nixpkgs(), + nixbuilder.BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + ) errz.Fatal(err) for _, e := range taskEnv { diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index a944607f..38712e8d 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -223,12 +223,19 @@ type BuildEnvironmentArgs struct { // List of dependencies imported from the // shell.nix file. Bob doesn't parse the imports // therefore it's required to pass all dependencies. + // + // TODO: let the hases be computes from the outside + // as BuilEnvironment is called for each task. + // TODO: + // TODO: + // TODO: NixShellImports []string } // BuildEnvironment is running nix-shell for a list of dependencies and fetch its whole environment // -// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' -E nixExpressionFromDeps +// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' --expr 'with import { }; [pkg1, pkg2]' +// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' shell.nix // // nix shell can be started with empty list of packages so this method works with empty deps as well func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { @@ -252,8 +259,6 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentAr err = BuildDependencies(deps, cache) errz.Fatal(err) - expression := nixExpression(deps, nixpkgs) - var arguments []string for _, envKey := range global.EnvWhitelist { if _, exists := os.LookupEnv(envKey); exists { @@ -261,9 +266,13 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentAr } } arguments = append(arguments, []string{"--command", "env"}...) - arguments = append(arguments, []string{"--expr", expression}...) + + // if shellDotNix is set, use it as the shell.nix file (must be at cmd's end) + // otherwise use the expression containing the packages. if shellDotNix != nil { arguments = append(arguments, *shellDotNix) + } else { + arguments = append(arguments, []string{"--expr", nixExpression(deps, nixpkgs)}...) } cmd := exec.Command("nix-shell", "--pure") From 22bfc963cfae1731a2a43511592813b67b90f4f2 Mon Sep 17 00:00:00 2001 From: equanox Date: Thu, 14 Mar 2024 20:52:41 +0100 Subject: [PATCH 5/5] wip caching --- bob/bobfile/bobfile.go | 4 +++ bob/nix-builder/nix_builder.go | 59 ++++++++++++++++++---------------- pkg/filehash/filehash.go | 13 ++++++++ pkg/nix/nix.go | 30 ++++++++++------- 4 files changed, 67 insertions(+), 39 deletions(-) diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index 5cc315f9..1508ed51 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -72,6 +72,10 @@ type Bobfile struct { // Shell specifies a shell.nix file as usually used by nix-shell. // This is mutualy exclusive with Dependencies. ShellDotNix string `yaml:"shell"` + // ShellImports specifies imports for the shell.nix file. + // This is necessary as Bob does not parse the shell.nix file. + // Therfore it can't infere the imports. + ShellDotNixImports []string `yaml:"shellImports"` // Nixpkgs specifies an optional nixpkgs source. Nixpkgs string `yaml:"nixpkgs"` diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index 87b59cfc..3b01ad30 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -2,8 +2,10 @@ package nixbuilder import ( "fmt" + "slices" "github.com/benchkram/bob/pkg/envutil" + "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/errz" "github.com/benchkram/bob/bob/bobfile" @@ -89,31 +91,24 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run } var shellDotNix *string + var shellDotNixHash *string if ag.ShellDotNix != "" { shellDotNix = &ag.ShellDotNix + + // When a folder is given instead of the direct path to shell.nix + // bob assumes that the folder contains a shell.nix file. + // TODO: Implement me. + + // Concat and sort files to ensure consistent hash + shellDotNixFiles := []string{ag.ShellDotNix} + shellDotNixFiles = append(shellDotNixFiles, ag.ShellDotNixImports...) + slices.Sort(shellDotNixFiles) + // Copute hash of shell.nix and its imports + hash, err := filehash.HashOfFiles(shellDotNixFiles...) + errz.Fatal(err) + + shellDotNixHash = &hash } - // if ag.Shell != "" { - // boblog.Log.Info(fmt.Sprintf("using shell file %s, ignoring dependencies", ag.Shell)) - // // hash shell file - // if !file.Exists(ag.Shell) { - // return usererror.Wrap(fmt.Errorf("shell file %s does not exist", ag.Shell)) - // } - - // nixShellEnv, err := nix.NixShell(ag.Shell) - // errz.Fatal(err) - - // hash, err := filehash.Hash(ag.Shell) - // errz.Fatal(err) - - // n.envStore[envutil.Hash(hash)] = nixShellEnv - // for _, name := range buildTasksInPipeline { - // t := ag.BTasks[name] - // t.SetEnvID(envutil.Hash(hash)) - // ag.BTasks[name] = t - // } - // // TODO: run tasks - // return nil - // } // Resolve nix storePaths from dependencies // and rewrite the affected tasks. @@ -132,7 +127,10 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run if _, ok := n.envStore[envutil.Hash(hash)]; !ok { nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, - BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + BuildEnvironmentArgs{ + ShellDotNix: shellDotNix, + ShellDotNixHash: shellDotNixHash, + }, ) errz.Fatal(err) n.envStore[envutil.Hash(hash)] = nixShellEnv @@ -161,7 +159,10 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run if _, ok := environmentCache[hash]; !ok { nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, - BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + BuildEnvironmentArgs{ + ShellDotNix: shellDotNix, + ShellDotNixHash: shellDotNixHash, + }, ) errz.Fatal(err) environmentCache[hash] = nixShellEnv @@ -180,16 +181,18 @@ func (n *NB) BuildDependencies(deps []nix.Dependency) error { } type BuildEnvironmentArgs struct { - ShellDotNix *string + ShellDotNix *string + ShellDotNixHash *string } // BuildEnvironment builds the environment with all nix deps func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { return nix.BuildEnvironment(deps, nixpkgs, nix.BuildEnvironmentArgs{ - Cache: n.cache, - ShellCache: n.shellCache, - ShellDotNix: args.ShellDotNix, + Cache: n.cache, + ShellCache: n.shellCache, + ShellDotNix: args.ShellDotNix, + ShellDotNixHash: args.ShellDotNixHash, }, ) } diff --git a/pkg/filehash/filehash.go b/pkg/filehash/filehash.go index 3e924791..c368ed32 100644 --- a/pkg/filehash/filehash.go +++ b/pkg/filehash/filehash.go @@ -52,3 +52,16 @@ func HashOfFile(path string) (string, error) { } return hex.EncodeToString(h.Sum()), nil } + +// HashOfFiles gives hash of multiple files content +func HashOfFiles(paths ...string) (string, error) { + h := New() + + for _, path := range paths { + err := h.AddFile(path) + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum()), nil +} diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index 38712e8d..78433a31 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -220,16 +220,10 @@ type BuildEnvironmentArgs struct { // Path to a shell.nix file. ShellDotNix *string - // List of dependencies imported from the - // shell.nix file. Bob doesn't parse the imports - // therefore it's required to pass all dependencies. - // - // TODO: let the hases be computes from the outside - // as BuilEnvironment is called for each task. - // TODO: - // TODO: - // TODO: - NixShellImports []string + + // ShellDotNixHash is the hash of the shell.nix file + // and all it's imports as given by the Bobfile. + ShellDotNixHash *string } // BuildEnvironment is running nix-shell for a list of dependencies and fetch its whole environment @@ -244,6 +238,7 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentAr var cache *Cache var shellCache *ShellCache var shellDotNix *string + var shellDotNixHash *string if args.Cache != nil { cache = args.Cache @@ -253,6 +248,9 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentAr } if args.ShellDotNix != nil { shellDotNix = args.ShellDotNix + if args.ShellDotNixHash != nil { + shellDotNixHash = args.ShellDotNixHash + } } // building dependencies with nix-build to display store paths to output @@ -284,7 +282,17 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentAr cmd.Stderr = &errBuf if shellCache != nil { - key, err := shellCache.GenerateKey(deps, cmd.String()) + + // generate a key for the shell environment + // In case of a shell.nix file an additional hash is + // added to the key to ensure that the environment is + // re-generated if the shell.nix file or its imports change. + cmdStr := cmd.String() + if shellDotNixHash != nil { + cmdStr += *shellDotNixHash + } + + key, err := shellCache.GenerateKey(deps, cmdStr) errz.Fatal(err) if dat, ok := shellCache.Get(key); ok {