diff --git a/bob/bobfile/bobfile.go b/bob/bobfile/bobfile.go index b6454621..1508ed51 100644 --- a/bob/bobfile/bobfile.go +++ b/bob/bobfile/bobfile.go @@ -65,8 +65,18 @@ type Bobfile struct { // RTasks run tasks RTasks bobrun.RunMap `yaml:"run"` + // Dependencies are nix packages used on a global scope. + // Mutually exclusive with Shell. ??Overwrites task based dependencies.?? Dependencies []string `yaml:"dependencies"` + // Shell specifies a shell.nix file as usually used by nix-shell. + // This is mutualy exclusive with Dependencies. + ShellDotNix string `yaml:"shell"` + // ShellImports specifies imports for the shell.nix file. + // This is necessary as Bob does not parse the shell.nix file. + // Therfore it can't infere the imports. + ShellDotNixImports []string `yaml:"shellImports"` + // Nixpkgs specifies an optional nixpkgs source. Nixpkgs string `yaml:"nixpkgs"` diff --git a/bob/build.go b/bob/build.go index 54089abd..fffa75d5 100644 --- a/bob/build.go +++ b/bob/build.go @@ -48,7 +48,7 @@ func (b *B) Build(ctx context.Context, taskName string) (err error) { } // AggregateWithNixDeps does aggregation together with evaluating nix dependecies. -// Nic dependencies are altering a tasks input hash. +// Nix dependencies are altering a tasks input hash. // Use this function for building `bob inspect` cmds. func (b *B) AggregateWithNixDeps(taskName string) (aggregate *bobfile.Bobfile, err error) { defer errz.Recover(&err) diff --git a/bob/nix-builder/nix_builder.go b/bob/nix-builder/nix_builder.go index 1ce21ddc..3b01ad30 100644 --- a/bob/nix-builder/nix_builder.go +++ b/bob/nix-builder/nix_builder.go @@ -2,8 +2,10 @@ package nixbuilder import ( "fmt" + "slices" "github.com/benchkram/bob/pkg/envutil" + "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/errz" "github.com/benchkram/bob/bob/bobfile" @@ -88,6 +90,26 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run return usererror.Wrap(fmt.Errorf("nix is not installed on your system. Get it from %s", nix.DownloadURl())) } + var shellDotNix *string + var shellDotNixHash *string + if ag.ShellDotNix != "" { + shellDotNix = &ag.ShellDotNix + + // When a folder is given instead of the direct path to shell.nix + // bob assumes that the folder contains a shell.nix file. + // TODO: Implement me. + + // Concat and sort files to ensure consistent hash + shellDotNixFiles := []string{ag.ShellDotNix} + shellDotNixFiles = append(shellDotNixFiles, ag.ShellDotNixImports...) + slices.Sort(shellDotNixFiles) + // Copute hash of shell.nix and its imports + hash, err := filehash.HashOfFiles(shellDotNixFiles...) + errz.Fatal(err) + + shellDotNixHash = &hash + } + // Resolve nix storePaths from dependencies // and rewrite the affected tasks. for _, name := range buildTasksInPipeline { @@ -104,7 +126,12 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run errz.Fatal(err) if _, ok := n.envStore[envutil.Hash(hash)]; !ok { - nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs) + nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, + BuildEnvironmentArgs{ + ShellDotNix: shellDotNix, + ShellDotNixHash: shellDotNixHash, + }, + ) errz.Fatal(err) n.envStore[envutil.Hash(hash)] = nixShellEnv } @@ -114,7 +141,7 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run } // FIXME: environment cache is a workaround... - // either use envSTore and adapt run tasks to use ist as well + // either use envSTore and adapt run tasks to use it as well // or remove run tasks entirely. environmentCache := make(map[string][]string) for _, name := range runTasksInPipeline { @@ -131,7 +158,12 @@ func (n *NB) BuildNixDependencies(ag *bobfile.Bobfile, buildTasksInPipeline, run errz.Fatal(err) if _, ok := environmentCache[hash]; !ok { - nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs) + nixShellEnv, err := n.BuildEnvironment(deps, ag.Nixpkgs, + BuildEnvironmentArgs{ + ShellDotNix: shellDotNix, + ShellDotNixHash: shellDotNixHash, + }, + ) errz.Fatal(err) environmentCache[hash] = nixShellEnv } @@ -148,9 +180,21 @@ func (n *NB) BuildDependencies(deps []nix.Dependency) error { return nix.BuildDependencies(deps, n.cache) } +type BuildEnvironmentArgs struct { + ShellDotNix *string + ShellDotNixHash *string +} + // BuildEnvironment builds the environment with all nix deps -func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string) (_ []string, err error) { - return nix.BuildEnvironment(deps, nixpkgs, n.cache, n.shellCache) +func (n *NB) BuildEnvironment(deps []nix.Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { + return nix.BuildEnvironment(deps, nixpkgs, + nix.BuildEnvironmentArgs{ + Cache: n.cache, + ShellCache: n.shellCache, + ShellDotNix: args.ShellDotNix, + ShellDotNixHash: args.ShellDotNixHash, + }, + ) } // Clean removes all cached nix dependencies diff --git a/cli/cmd_inspect.go b/cli/cmd_inspect.go index dbfb4099..0a81c620 100644 --- a/cli/cmd_inspect.go +++ b/cli/cmd_inspect.go @@ -8,6 +8,7 @@ import ( "sort" "github.com/benchkram/bob/bob" + nixbuilder "github.com/benchkram/bob/bob/nix-builder" "github.com/benchkram/bob/pkg/boblog" "github.com/benchkram/bob/pkg/filehash" "github.com/benchkram/bob/pkg/usererror" @@ -77,7 +78,13 @@ func runEnv(taskname string) { } task = bobfile.BTasks[taskname] - taskEnv, err := b.Nix().BuildEnvironment(task.Dependencies(), task.Nixpkgs()) + var shellDotNix *string + if bobfile.ShellDotNix != "" { + shellDotNix = &bobfile.ShellDotNix + } + taskEnv, err := b.Nix().BuildEnvironment(task.Dependencies(), task.Nixpkgs(), + nixbuilder.BuildEnvironmentArgs{ShellDotNix: shellDotNix}, + ) errz.Fatal(err) for _, e := range taskEnv { diff --git a/pkg/filehash/filehash.go b/pkg/filehash/filehash.go index 3e924791..c368ed32 100644 --- a/pkg/filehash/filehash.go +++ b/pkg/filehash/filehash.go @@ -52,3 +52,16 @@ func HashOfFile(path string) (string, error) { } return hex.EncodeToString(h.Sum()), nil } + +// HashOfFiles gives hash of multiple files content +func HashOfFiles(paths ...string) (string, error) { + h := New() + + for _, path := range paths { + err := h.AddFile(path) + if err != nil { + return "", err + } + } + return hex.EncodeToString(h.Sum()), nil +} diff --git a/pkg/nix/nix.go b/pkg/nix/nix.go index 34c260e6..78433a31 100644 --- a/pkg/nix/nix.go +++ b/pkg/nix/nix.go @@ -211,20 +211,52 @@ func source(nixpkgs string) string { return "" } +type BuildEnvironmentArgs struct { + + // Cache is used to store the store path of the nix dependencies + Cache *Cache + // ShellCache is used to store the environment of a nix-shell command + ShellCache *ShellCache + + // Path to a shell.nix file. + ShellDotNix *string + + // ShellDotNixHash is the hash of the shell.nix file + // and all it's imports as given by the Bobfile. + ShellDotNixHash *string +} + // BuildEnvironment is running nix-shell for a list of dependencies and fetch its whole environment // -// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' -E nixExpressionFromDeps +// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' --expr 'with import { }; [pkg1, pkg2]' +// nix-shell --pure --keep NIX_SSL_CERT_FILE --keep SSL_CERT_FILE -p --command 'env' shell.nix // // nix shell can be started with empty list of packages so this method works with empty deps as well -func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCache *ShellCache) (_ []string, err error) { +func BuildEnvironment(deps []Dependency, nixpkgs string, args BuildEnvironmentArgs) (_ []string, err error) { defer errz.Recover(&err) + var cache *Cache + var shellCache *ShellCache + var shellDotNix *string + var shellDotNixHash *string + + if args.Cache != nil { + cache = args.Cache + } + if args.ShellCache != nil { + shellCache = args.ShellCache + } + if args.ShellDotNix != nil { + shellDotNix = args.ShellDotNix + if args.ShellDotNixHash != nil { + shellDotNixHash = args.ShellDotNixHash + } + } + // building dependencies with nix-build to display store paths to output err = BuildDependencies(deps, cache) errz.Fatal(err) - expression := nixExpression(deps, nixpkgs) - var arguments []string for _, envKey := range global.EnvWhitelist { if _, exists := os.LookupEnv(envKey); exists { @@ -232,7 +264,14 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCach } } arguments = append(arguments, []string{"--command", "env"}...) - arguments = append(arguments, []string{"--expr", expression}...) + + // if shellDotNix is set, use it as the shell.nix file (must be at cmd's end) + // otherwise use the expression containing the packages. + if shellDotNix != nil { + arguments = append(arguments, *shellDotNix) + } else { + arguments = append(arguments, []string{"--expr", nixExpression(deps, nixpkgs)}...) + } cmd := exec.Command("nix-shell", "--pure") cmd.Args = append(cmd.Args, arguments...) @@ -243,7 +282,17 @@ func BuildEnvironment(deps []Dependency, nixpkgs string, cache *Cache, shellCach cmd.Stderr = &errBuf if shellCache != nil { - key, err := shellCache.GenerateKey(deps, cmd.String()) + + // generate a key for the shell environment + // In case of a shell.nix file an additional hash is + // added to the key to ensure that the environment is + // re-generated if the shell.nix file or its imports change. + cmdStr := cmd.String() + if shellDotNixHash != nil { + cmdStr += *shellDotNixHash + } + + key, err := shellCache.GenerateKey(deps, cmdStr) errz.Fatal(err) if dat, ok := shellCache.Get(key); ok { @@ -336,3 +385,43 @@ func HashDependencies(deps []Dependency) (_ string, err error) { } return string(h.Sum()), nil } + +// NixShell returns the environment of a nix-shell command +func NixShell(path string) ([]string, error) { + + args := []string{} + for _, envKey := range global.EnvWhitelist { + if _, exists := os.LookupEnv(envKey); exists { + args = append(args, []string{"--keep", envKey}...) + } + } + args = append(args, []string{"--pure", "--command", "env"}...) + args = append(args, path) + cmd := exec.Command("nix-shell", args...) + boblog.Log.V(5).Info(fmt.Sprintf("Executing command:\n %s", cmd.String())) + + var out bytes.Buffer + var errBuf bytes.Buffer + cmd.Stdout = &out + cmd.Stderr = &errBuf + + err := cmd.Run() + if err != nil { + return nil, prepareRunError(err, cmd.String(), errBuf) + } + + // if NIX_SSL_CERT_FILE && SSL_CERT_FILE are set to /no-cert-file.crt unset them + var clearedEnv []string + for _, e := range strings.Split(out.String(), "\n") { + pair := strings.SplitN(e, "=", 2) + if pair[0] == "NIX_SSL_CERT_FILE" && pair[1] == "/no-cert-file.crt" { + continue + } + if pair[0] == "SSL_CERT_FILE" && pair[1] == "/no-cert-file.crt" { + continue + } + clearedEnv = append(clearedEnv, e) + } + + return clearedEnv, nil +}