Add git as a built-in external dep

all.bramble

@@ -10,6 +10,7 @@ load("github.com/maxmcd/bramble/tests/nested-sources/another-folder/nested")
 def all():
     return [
         lib.busybox(),
+        lib.git_fetcher(),
         nested.nested(),
         simple.simple(),
     ]

bramble.lock

@@ -1,7 +1,9 @@
 [URLHashes]
+  "http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz" = "a360bf615041545b07f156b5819d0d407ff59b160d5a6f0312a291624ef935e5"
   "http://s.minos.io/archive/bifrost/x86_64/git-2.7.2-2.tar.gz" = "654146e3a239fef2e79511bf596098b36ebfe583f28cd0dea0c3f5f52f1a89d5"
   "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"
   "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "2ae410370b8e9113968ffa6e52f38eea7f17df5f436bd6a69cc41c6ca01541a1"
+  "https://brmbl.s3.amazonaws.com/ca-certificates.crt" = "c61418dc5563f4498af272feab598ef7e1ee0f81babfcdd21c7f344894283439"
   "https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "22fadd82e935b2f081ba07ab9a9c7ce174d10c237cc95182c4ed14c8c276e06a"
   "https://brmbl.s3.amazonaws.com/patchelf.tar.gz" = "67ee6623207754a18d81624d630d9addbf6234ab1e6c44ddba9179621720f960"
   "https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz" = "f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52"

lib/default.bramble

@@ -1,8 +1,12 @@
 load("github.com/maxmcd/bramble/lib/std")
 
 
+def cacerts():
+    return std.fetch_url("https://brmbl.s3.amazonaws.com/ca-certificates.crt")
+
+
 def git():
-    b = std.fetch_url("http://s.minos.io/archive/bifrost/x86_64/git-2.7.2-2.tar.gz")
+    b = std.fetch_url("http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz")
     return derivation(
         name="git",
         builder=busybox().out + "/bin/sh",
@@ -20,6 +24,29 @@ def git():
     )
 
 
+def git_fetcher():
+    return derivation(
+        name="git_fetcher",
+        builder=busybox().out + "/bin/sh",
+        args=["-c", ""],  # noop
+        env=dict(
+            git=git(),
+            PATH=git().out + "/bin",
+            GIT_EXEC_PATH=git().out + "/libexec/git-core",
+            GIT_SSL_CAINFO=cacerts(),
+        ),
+    )
+
+
+# Need better sandboxing
+# def git_test():
+#     return derivation(
+#         "git-test",
+#         "fetch_git",
+#         env=dict(url="https://github.com/maxmcd/bramble.git", cachebust=1),
+#     )
+
+
 def zig():
     b = std.fetch_url(
         "https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.946+6237dc0ab.tar.xz"

notes/29-built-in-external-dependencies.md

@@ -2,4 +2,9 @@
 We should remove mholt/achiver and any other built-in software that we should more reasonably. Now that build and project are separatated we can just define the derivations we would need in the input format that the build expects. We can feed in derivation inputs and then write our source into the store (if needed). These derivations can download programs like git and tar at fixed versions. These dependencies will show up in the bramble.toml file so that a project can be guaranteed to see the same versions of git/tar/whatever to rebuild the software that they've fetched
 
 
-We can even have the derivation for older versions of bramble. When a project needs an old version of bramble we just store all old versions of bramble to download and run (this is trivial now that we don't need root). Project specifies a specific version of bramble we download and run that instead (another reason to keep the binary small).
+We can even have the derivation for older versions of bramble. When a project needs an old version of bramble we just store all old versions of bramble to download and run (this is trivial now that we don't need root). Project specifies a specific version of bramble we download and run that instead (another reason to keep the binary small). Ah, could even keep libcontainer just in a sandbox utility that we download to run builds. Oof, interesting idea.
+
+
+Write a tool that takes starlark derivations and turns them into go code. Generate the resulting derivations and then output go code with the structs.
+
+

notes/30-file-hash-tree-efficiency.md

@@ -0,0 +1,12 @@
+
+File copying and hashing is expensive. Could hash each file and assemble in a tree. When an individual file is changed we just hash that file and then re-hash the file hash values. This would make incremental changes less painful. The state of the previous hash calculation would need to sit on disk somewhere so that we don't have to re-calculate those hashes every time.
+
+Wait, we could do this for all file folders, and store the file-maps remotely. That way if you pull a dependency you can run it immediately and pull the files down as you need them. We could do that with https://github.com/google/crfs/blob/master/crfs.go so that we can get fast starting downloads. On linux at least. Would need to have osxfuse installed for it to work on osx.
+
+So, not sure where this should go. These are two thoughts tho, one is about using a tree to hash, the other is about allowing one to retriteve their contents.
+
+We should try out the first one though, would make things fast.
+
+Ah man, this also means you can start building immediately. If you use the fuse layer to serve the files then you can start building before the sources are tarred up and sent out. You can finish building before the build state arrives.
+
+Would need to be able to disable it in the derivation definition.

pkg/bramble/build.go

@@ -10,35 +10,45 @@ import (
 	"github.com/pkg/errors"
 )
 
-func runBuild(command string, args []string) error {
+func runBuildFromOutput(output project.ExecModuleOutput) (outputDerivations []build.Derivation, err error) {
+	return runBuild(func(p *project.Project) (project.ExecModuleOutput, error) {
+		return output, nil
+	})
+}
+
+func runBuildFromCLI(command string, args []string) (outputDerivations []build.Derivation, err error) {
+	return runBuild(func(p *project.Project) (output project.ExecModuleOutput, err error) {
+		return p.ExecModule(project.ExecModuleInput{
+			Command:   command,
+			Arguments: args,
+		})
+	})
+}
+
+func runBuild(execModule func(*project.Project) (project.ExecModuleOutput, error)) (outputDerivations []build.Derivation, err error) {
 	p, err := project.NewProject(".")
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	output, err := p.ExecModule(project.ExecModuleInput{
-		Command:   command,
-		Arguments: args,
-	})
+	output, err := execModule(p)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	store, err := build.NewStore("")
 	if err != nil {
-		return err
+		return nil, err
 	}
+	store.RegisterGetGit(getGit)
 
 	builder := store.NewBuilder(false, p.URLHashes())
 
 	derivationIDUpdates := map[project.Dependency]build.DerivationOutput{}
-	allDerivations := []project.Derivation{}
+	// allDerivations := []build.Derivation{}
 	derivationDataLock := sync.Mutex{}
 
 	err = output.WalkAndPatch(1, func(dep project.Dependency, drv project.Derivation) (buildOutputs []project.BuildOutput, err error) {
-		derivationDataLock.Lock()
-		allDerivations = append(allDerivations, drv)
-		derivationDataLock.Unlock()
 		inputDerivations := []build.DerivationOutput{}
 
 		derivationDataLock.Lock()
@@ -73,9 +83,11 @@ func runBuild(command string, args []string) error {
 		if buildDrv, _, err = builder.BuildDerivation(context.Background(), buildDrv); err != nil {
 			return nil, err
 		}
+
+		derivationDataLock.Lock()
+		// allDerivations = append(allDerivations, buildDrv)
 		// Store the derivation outputs in the map for reference when building
 		// input derivations later. Also populate the buildOutputs
-		derivationDataLock.Lock()
 		for i, o := range buildDrv.OutputNames {
 			out := buildDrv.Outputs[i]
 			derivationIDUpdates[project.Dependency{
@@ -91,16 +103,21 @@ func runBuild(command string, args []string) error {
 				OutputPath: build.BramblePrefixOfRecord + "/" + out.Path,
 			})
 		}
+		for hash := range output.Output {
+			if hash == dep.Hash {
+				outputDerivations = append(outputDerivations, buildDrv)
+			}
+		}
 		derivationDataLock.Unlock()
 		return
 	})
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	err = p.AddURLHashesToLockfile(builder.URLHashes)
 	if err != nil {
-		return err
+		return outputDerivations, err
 	}
-	return nil
+	return outputDerivations, err
 }

pkg/bramble/cli.go

@@ -29,7 +29,7 @@ func createAndParseCLI(args []string) (*ffcli.Command, error) {
 			ShortUsage: "bramble build [options] [module]:<function> [args...]",
 			ShortHelp:  "Build a function",
 			LongHelp:   "Build a function",
-			Exec:       func(ctx context.Context, args []string) error { err := buildCommand(ctx, args, false); return err },
+			Exec:       func(ctx context.Context, args []string) error { _, err := runBuildFromCLI("build", args); return err },
 		},
 		{
 			Name:       "shell",
@@ -44,15 +44,22 @@ func createAndParseCLI(args []string) (*ffcli.Command, error) {
 			ShortHelp:  "Run an interactive shell",
 			Exec:       func(ctx context.Context, args []string) error { return repl(args) },
 		},
+		{
+			Name:       "print-build-input",
+			ShortUsage: "bramble print-build-input",
+			Exec:       func(ctx context.Context, args []string) error { return printBuildInput(args) },
+		},
 		{
 			Name:       "derivation",
 			ShortUsage: "bramble derivation <subcomand>",
 			ShortHelp:  "Work with derivations directly",
-			Subcommands: []*ffcli.Command{{
-				Name:       "build",
-				ShortUsage: "bramble derivation build ~/bramble/store/3orpqhjdgtvfbqbhpecro3qe6heb3jvq-simple.drv",
-				Exec:       func(ctx context.Context, args []string) error { return nil },
-			}},
+			Subcommands: []*ffcli.Command{
+				{
+					Name:       "build",
+					ShortUsage: "bramble derivation build ~/bramble/store/3orpqhjdgtvfbqbhpecro3qe6heb3jvq-simple.drv",
+					Exec:       func(ctx context.Context, args []string) error { return nil },
+				},
+			},
 			Exec: func(ctx context.Context, args []string) error { return flag.ErrHelp },
 		},
 		{
@@ -66,7 +73,7 @@ func createAndParseCLI(args []string) (*ffcli.Command, error) {
 
 	// storeGC = &ffcli.Command{
 	// 	Name:       "gc",
-	// 	ShortUsage: "bramble store gc",
+	// 	ShortUsage: "bramble gc",
 	// 	ShortHelp:  "Run the bramble garbage collector against the store",
 	// 	LongHelp: `    Collect garbage
 
@@ -81,7 +88,7 @@ func createAndParseCLI(args []string) (*ffcli.Command, error) {
 
 	// storeAudit = &ffcli.Command{
 	// 	Name:       "audit",
-	// 	ShortUsage: "bramble store audit",
+	// 	ShortUsage: "bramble verify||check", #TODO
 	// 	ShortHelp:  "",
 	// 	LongHelp:   "",
 	// 	Exec: func(ctx context.Context, args []string) error {
@@ -189,10 +196,6 @@ func countFlags(fs *flag.FlagSet) (n int) {
 	return n
 }
 
-func buildCommand(ctx context.Context, args []string, rootLess bool) error {
-	return runBuild("build", args)
-}
-
 func shell(ctx context.Context, args []string) (err error) {
 	return
 }

pkg/bramble/git.go

@@ -0,0 +1,24 @@
+package bramble
+
+import (
+	"encoding/json"
+
+	build "github.com/maxmcd/bramble/pkg/bramblebuild"
+	project "github.com/maxmcd/bramble/pkg/brambleproject"
+)
+
+func getGit() (drv build.Derivation, err error) {
+	execOutputJSON := `{"Output":{"mfmcau5igf7rbmkkftrkmt7y22vbyavs":{"Args":["-c",""],"Builder":"{{ dsvr3dv6wjmcwcynbwb2fkd55rmukci5:out }}/bin/sh","Dependencies":[{"Hash":"23kutq6mumi6pnb4dbkt44vkgca73yiv","Output":"out"},{"Hash":"dsvr3dv6wjmcwcynbwb2fkd55rmukci5","Output":"out"},{"Hash":"gwnymix2vvo3mnh2pqfupvbflgcmzqdk","Output":"out"}],"Env":{"GIT_EXEC_PATH":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}/libexec/git-core","GIT_SSL_CAINFO":"{{ 23kutq6mumi6pnb4dbkt44vkgca73yiv:out }}","PATH":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}/bin","git":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}"},"Name":"git_fetcher","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}}},"AllDerivations":{"23kutq6mumi6pnb4dbkt44vkgca73yiv":{"Args":null,"Builder":"fetch_url","Dependencies":null,"Env":{"url":"https://brmbl.s3.amazonaws.com/ca-certificates.crt"},"Name":"ca-certificates.crt","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}},"2oscpeuv4wsscndhcbxero2sabmdbzn4":{"Args":null,"Builder":"fetch_url","Dependencies":null,"Env":{"url":"https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz"},"Name":"busybox-x86_64.tar.gz","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}},"dsvr3dv6wjmcwcynbwb2fkd55rmukci5":{"Args":["sh","-c","\n    set -e\n    $busybox_download/busybox-x86_64 mkdir $out/bin\n    $busybox_download/busybox-x86_64 cp $busybox_download/busybox-x86_64 $out/bin/busybox\n    cd $out/bin\n    for command in $(./busybox --list); do\n        ./busybox ln -s busybox $command\n    done\n    "],"Builder":"{{ 2oscpeuv4wsscndhcbxero2sabmdbzn4:out }}/busybox-x86_64","Dependencies":[{"Hash":"2oscpeuv4wsscndhcbxero2sabmdbzn4","Output":"out"}],"Env":{"busybox_download":"{{ 2oscpeuv4wsscndhcbxero2sabmdbzn4:out }}"},"Name":"busybox","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}},"gwnymix2vvo3mnh2pqfupvbflgcmzqdk":{"Args":["-c","\n            set -ex\n            cp -r $src/usr/* $out\n            mkdir test\n            cd test\n            $out/bin/git --version\n            "],"Builder":"{{ dsvr3dv6wjmcwcynbwb2fkd55rmukci5:out }}/bin/sh","Dependencies":[{"Hash":"dsvr3dv6wjmcwcynbwb2fkd55rmukci5","Output":"out"},{"Hash":"wcunj6ugkakaqrejz77jbfoojqxgwzou","Output":"out"}],"Env":{"PATH":"{{ dsvr3dv6wjmcwcynbwb2fkd55rmukci5:out }}/bin","src":"{{ wcunj6ugkakaqrejz77jbfoojqxgwzou:out }}"},"Name":"git","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}},"mfmcau5igf7rbmkkftrkmt7y22vbyavs":{"Args":["-c",""],"Builder":"{{ dsvr3dv6wjmcwcynbwb2fkd55rmukci5:out }}/bin/sh","Dependencies":[{"Hash":"23kutq6mumi6pnb4dbkt44vkgca73yiv","Output":"out"},{"Hash":"dsvr3dv6wjmcwcynbwb2fkd55rmukci5","Output":"out"},{"Hash":"gwnymix2vvo3mnh2pqfupvbflgcmzqdk","Output":"out"}],"Env":{"GIT_EXEC_PATH":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}/libexec/git-core","GIT_SSL_CAINFO":"{{ 23kutq6mumi6pnb4dbkt44vkgca73yiv:out }}","PATH":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}/bin","git":"{{ gwnymix2vvo3mnh2pqfupvbflgcmzqdk:out }}"},"Name":"git_fetcher","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}},"wcunj6ugkakaqrejz77jbfoojqxgwzou":{"Args":null,"Builder":"fetch_url","Dependencies":null,"Env":{"url":"http://s.minos.io/archive/bifrost/x86_64/git-2.10.2-1.tar.gz"},"Name":"git-2.10.2-1.tar.gz","Outputs":["out"],"Platform":"","Sources":{"Files":null,"Location":""}}}}`
+
+	var output project.ExecModuleOutput
+
+	if err = json.Unmarshal([]byte(execOutputJSON), &output); err != nil {
+		return
+	}
+	drvs, err := runBuildFromOutput(output)
+	if err != nil {
+		return
+	}
+
+	return drvs[0], nil
+}

pkg/bramble/integration_test.go

@@ -2,7 +2,6 @@ package bramble
 
 import (
 	"bufio"
-	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -121,7 +120,7 @@ func TestIntegrationRunAlmostAllPublicFunctions(t *testing.T) {
 				}
 			}
 			if !t.Run(module, func(t *testing.T) {
-				if err := buildCommand(context.Background(), []string{module}, true); err != nil {
+				if _, err := runBuildFromCLI("test", []string{module}); err != nil {
 					t.Fatal(starutil.AnnotateError(err))
 				}
 			}) {

pkg/bramble/print_build_input.go

@@ -0,0 +1,23 @@
+package bramble
+
+import (
+	"encoding/json"
+	"fmt"
+
+	project "github.com/maxmcd/bramble/pkg/brambleproject"
+)
+
+func printBuildInput(args []string) error {
+	p, err := project.NewProject(".")
+	if err != nil {
+		return err
+	}
+
+	output, err := p.ExecModule(project.ExecModuleInput{
+		Command:   "print-build-args",
+		Arguments: args,
+	})
+	b, err := json.Marshal(output)
+	fmt.Println(string(b))
+	return nil
+}

pkg/bramblebuild/builder.go

@@ -102,6 +102,8 @@ func (b *Builder) buildDerivation(ctx context.Context, drv Derivation, shell boo
 	switch drv.Builder {
 	case "fetch_url":
 		err = b.fetchURLBuilder(ctx, drvCopy, outputPaths)
+	case "fetch_git":
+		err = b.fetchGitBuilder(ctx, drvCopy, outputPaths)
 	default:
 		err = b.regularBuilder(ctx, drvCopy, buildDir, outputPaths, shell)
 	}
@@ -154,12 +156,48 @@ func (b *Builder) hashAndMoveFetchURL(ctx context.Context, drv Derivation, outpu
 	return outputs, err
 }
 
+func (b *Builder) fetchGitBuilder(ctx context.Context, drv Derivation, outputPaths map[string]string) (err error) {
+	gitDrv, err := b.store.getGit()
+	if err != nil {
+		return errors.Wrap(err, "error trying to install git")
+	}
+	gitDrv.store = b.store // let the other store be gc'd
+
+	if _, ok := outputPaths["out"]; len(outputPaths) > 1 || !ok {
+		return errors.New("the fetch_url builder can only have the defalt output \"out\"")
+	}
+	url, ok := drv.Env["url"]
+	if !ok {
+		return errors.New("fetch_url requires the environment variable 'url' to be set")
+	}
+	// derivation can provide a hash, but usually this is just in the lockfile
+	hash := drv.Env["hash"]
+
+	outputPath := outputPaths["out"]
+
+	gitDir := gitDrv.Env["git"]
+	cmd := exec.Command(filepath.Join(gitDir, "/bin/git"), "clone", url, outputPath)
+	for k, v := range drv.Env {
+		if k == "PATH" {
+			v = os.Getenv("PATH") + ":" + v
+		}
+		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+	}
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return err
+	}
+	_ = hash
+	return nil
+}
+
 func (b *Builder) fetchURLBuilder(ctx context.Context, drv Derivation, outputPaths map[string]string) (err error) {
 	region := trace.StartRegion(ctx, "fetchURLBuilder")
 	defer region.End()
 
 	if _, ok := outputPaths["out"]; len(outputPaths) > 1 || !ok {
-		return errors.New("the fetchurl builtin can only have the defalt output \"out\"")
+		return errors.New("the fetch_url builder can only have the defalt output \"out\"")
 	}
 	url, ok := drv.Env["url"]
 	if !ok {
@@ -231,6 +269,13 @@ func (b *Builder) downloadFile(ctx context.Context, url string, hash string) (pa
 		TLSHandshakeTimeout:   10 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 	}
+
+	// TODO: this is prob not ok, it's sort of fine for us but hostile against
+	// custom certs. maybe an easy way to fix that? Should just accept available
+	// certs (unless repro???). I think this was also motivated by barebones
+	// docker env and we can just mount in known cert locations, right?
+	//
+	// also yes, we have to make certs work for other software, so surely they can work for us
 	certPool, err := gocertifi.CACerts()
 	transport.TLSClientConfig = &tls.Config{RootCAs: certPool}