Parameterize working directory (#23)

* Start of wd changes

* remove chdir

* fix maxmcd hash

* more restructuring

* fit integration tests

* move build tests to standalone package

* vscode settings

* add options

* consolidate test tmpdir fn

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
         uses: actions/checkout@v2
       - name: Run all tests
         run: |
-          make ci_test
+          make -j ci_test
       # for your debugging needs
       # - name: Setup tmate session
       #   uses: mxschmitt/action-tmate@v3

.gitignore

@@ -1,6 +1,7 @@
 brambles
 *.gz
-.vscode
+.vscode/*
+!.vscode/settings.json
 .idea
 *.prof
 trace.out

.vscode/settings.json

@@ -0,0 +1,9 @@
+{
+    "files.associations": {
+        "*.bramble": "python",
+    },
+    "python.formatting.provider": "black",
+    "python.linting.flake8Enabled": true,
+    "python.linting.enabled": true,
+    "python.languageServer": "None"
+}

Makefile

@@ -9,7 +9,7 @@ ci_test: go_ci_test \
 gotestsum:
 	go get gotest.tools/gotestsum
 
-go_ci_test: gotestsum
+go_ci_test: install gotestsum
 	gotestsum -- -race -v ./...
 
 go_test:

bramble.lock

@@ -1,7 +1,8 @@
 [URLHashes]
   "http://tarballs.nixos.org/stdenv-linux/x86_64/c5aabb0d603e2c1ea05f5a93b3be82437f5ebf31/bootstrap-tools.tar.xz" = "a5ce9c155ed09397614646c9717fc7cd94b1023d7b76b618d409e4fefd6e9d39"
   "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz" = "2ae410370b8e9113968ffa6e52f38eea7f17df5f436bd6a69cc41c6ca01541a1"
+  "https://brmbl.s3.amazonaws.com/file-links.tar.gz" = "22fadd82e935b2f081ba07ab9a9c7ce174d10c237cc95182c4ed14c8c276e06a"
   "https://brmbl.s3.amazonaws.com/patchelf.tar.gz" = "67ee6623207754a18d81624d630d9addbf6234ab1e6c44ddba9179621720f960"
   "https://dl.google.com/go/go1.4-bootstrap-20171003.tar.gz" = "f4ff5b5eb3a3cae1c993723f3eab519c5bae18866b5e5f96fe1102f0cb5c3e52"
-  "https://maxmcd.com/" = "b17315f3eac25dc23a59cc0ec2820c78a0b9890f7fea5a44deaef5a3c6cd9e59"
+  "https://maxmcd.com/" = "c0c11d3923e78f8c103985e2035df7b146e9946bf7eabcd38d71459d94063a7a"
   "https://ziglang.org/builds/zig-linux-x86_64-0.8.0-dev.1431+c760532be.tar.xz" = "22e83ec5f2483bb0d9a861346c883b4b2f7f572595f3453d1131b31898e98a33"

go.mod

@@ -8,10 +8,10 @@ require (
 	github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054
 	github.com/containerd/console v1.0.0
 	github.com/creack/pty v1.1.11
-	github.com/davecgh/go-spew v1.1.1
 	github.com/docker/docker v1.4.2-0.20191101170500-ac7306503d23
 	github.com/fsouza/go-dockerclient v1.6.5
 	github.com/go-git/go-git/v5 v5.3.0
+	github.com/google/go-cmp v0.5.5 // indirect
 	github.com/jaguilar/vt100 v0.0.0-20201024211400-81de19cb81a4
 	github.com/maxmcd/dag v0.0.0-20210316172417-f02e4b03c6e9
 	github.com/mholt/archiver/v3 v3.3.1-0.20200626164424-d44471c49aa7
@@ -23,6 +23,7 @@ require (
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
 	go.starlark.net v0.0.0-20200901195727-6e684ef5eeee
 	go.uber.org/zap v1.10.0
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
 	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 )

go.sum

@@ -198,8 +198,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -545,8 +546,9 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

notes/24-early-cuttoff-optimization.md

@@ -2,7 +2,7 @@ Nix handles early cutoff with fixed output derivations. These must be manually k
 
 Because the dependency graph in Bramble is (currently) constructed using the hash of input derivations any change to build input will result in a full rebuild.
 
-I think it's important when thinking about a solution here to remember that all build inputs are either filesystem files or `fetch` derivations, so don't get too creative about storing build state.
+When thinking about solutions here remember that all build inputs are either filesystem files or `fetch` derivations, so don't get too creative about storing build state.
 
 ### Always use the build output as the derivation hash
 

pkg/bramble/bramble.go

@@ -71,6 +71,8 @@ var (
 	BrambleExtension string = ".bramble"
 )
 
+// Bramble is the main bramble client. It has various caches and metadata
+// associated with running bramble.
 type Bramble struct {
 	thread      *starlark.Thread
 	predeclared starlark.StringDict
@@ -83,6 +85,9 @@ type Bramble struct {
 	lockFile       LockFile
 	lockFileLock   sync.Mutex
 
+	// working directory
+	wd string
+
 	derivationFn *derivationFunction
 
 	store store.Store
@@ -381,6 +386,9 @@ func (b *Bramble) archiveAndScanOutputDirectory(ctx context.Context, tarOutput,
 }
 
 func (b *Bramble) moduleNameFromFileName(filename string) (moduleName string, err error) {
+	if !filepath.IsAbs(filename) {
+		filename = filepath.Join(b.wd, filename)
+	}
 	filename, err = filepath.Abs(filename)
 	if err != nil {
 		return "", err
@@ -830,24 +838,43 @@ func (b *Bramble) buildDerivationOutputs(ctx context.Context, dos DerivationOutp
 	return err
 }
 
-func (b *Bramble) init(wd string, expectProject bool) (err error) {
+// Option can be used to set various options when initializing bramble
+type Option func(*Bramble)
+
+// OptionNoRoot ensures bramble don't use features that require root like setuid binaries
+func OptionNoRoot(b *Bramble) {
+	b.noRoot = true
+}
+
+// NewBramble creates a new bramble instance. If the working directory passed is
+// within a bramble project that projects configuration will be laoded
+func NewBramble(wd string, opts ...Option) (b *Bramble, err error) {
+	b = &Bramble{}
 	b.moduleCache = map[string]string{}
 	b.filenameCache = NewBiStringMap()
 	b.derivations = &DerivationsMap{}
+	b.wd = wd
+
+	for _, opt := range opts {
+		opt(b)
+	}
+
+	// Make b.wd absolute
+	if !filepath.IsAbs(b.wd) {
+		wd, _ := os.Getwd()
+		b.wd = filepath.Join(wd, b.wd)
+	}
 
 	if b.store.IsEmpty() {
 		if b.store, err = store.NewStore(); err != nil {
-			return err
+			return
 		}
 	}
 
-	found, loc := findConfig(wd)
-	if expectProject && !found {
-		return errors.New("couldn't find a bramble.toml file in this directory or any parent")
-	}
+	found, loc := findConfig(b.wd)
 	if found {
 		if err := b.loadConfig(loc); err != nil {
-			return err
+			return nil, err
 		}
 	}
 
@@ -856,7 +883,7 @@ func (b *Bramble) init(wd string, expectProject bool) (err error) {
 		Load: b.load,
 	}
 
-	return b.initPredeclared()
+	return b, b.initPredeclared()
 }
 
 func (b *Bramble) initPredeclared() (err error) {
@@ -885,8 +912,8 @@ func (b *Bramble) load(thread *starlark.Thread, module string) (globals starlark
 }
 
 func findAllDerivationsInProject(loc string) (derivations []*Derivation, err error) {
-	b := Bramble{}
-	if err := b.init(loc, true); err != nil {
+	b, err := NewBramble(loc)
+	if err != nil {
 		return nil, err
 	}
 
@@ -926,8 +953,7 @@ func findAllDerivationsInProject(loc string) (derivations []*Derivation, err err
 }
 
 func (b *Bramble) execTestFileContents(script string) (v starlark.Value, err error) {
-	wd, _ := os.Getwd()
-	globals, err := starlark.ExecFile(b.thread, filepath.Join(wd, "foo.bramble"), script, b.predeclared)
+	globals, err := starlark.ExecFile(b.thread, filepath.Join(b.wd, "foo.bramble"), script, b.predeclared)
 	if err != nil {
 		return nil, err
 	}
@@ -1040,9 +1066,6 @@ func (b *Bramble) starlarkExecFile(moduleName, filename string) (globals starlar
 }
 
 func (b *Bramble) repl(_ []string) (err error) {
-	if err := b.init(".", false); err != nil {
-		return err
-	}
 	repl.REPL(b.thread, b.predeclared)
 	return nil
 }
@@ -1054,10 +1077,6 @@ func (b *Bramble) parseAndCallBuildArg(cmd string, args []string) (derivations [
 		return
 	}
 
-	if err = b.init(".", true); err != nil {
-		return
-	}
-
 	// parse something like ./tests:foo into the correct module and function
 	// name
 	module, fn, err := b.parseModuleFuncArgument(args)
@@ -1089,7 +1108,10 @@ func (b *Bramble) parseAndCallBuildArg(cmd string, args []string) (derivations [
 	return
 }
 
-func (b *Bramble) shell(ctx context.Context, args []string) (err error) {
+func (b *Bramble) Shell(ctx context.Context, args []string) (err error) {
+	if !b.withinProject() {
+		return ErrNotInProject
+	}
 	derivations, err := b.parseAndCallBuildArg("build", args)
 	if err != nil {
 		return err
@@ -1115,7 +1137,14 @@ func (b *Bramble) shell(ctx context.Context, args []string) (err error) {
 	return nil
 }
 
-func (b *Bramble) build(ctx context.Context, args []string) (err error) {
+func (b *Bramble) withinProject() bool {
+	return b.configLocation != ""
+}
+
+func (b *Bramble) Build(ctx context.Context, args []string) (err error) {
+	if !b.withinProject() {
+		return ErrNotInProject
+	}
 	derivations, err := b.parseAndCallBuildArg("build", args)
 	if err != nil {
 		return err
@@ -1143,10 +1172,7 @@ func valuesToDerivations(values starlark.Value) (derivations []*Derivation) {
 	return
 }
 
-func (b *Bramble) gc(_ []string) (err error) {
-	if err = b.init(".", false); err != nil {
-		return
-	}
+func (b *Bramble) GC(_ []string) (err error) {
 	derivations, err := b.collectDerivationsToPreserve()
 	if err != nil {
 		return
@@ -1313,11 +1339,8 @@ func (b *Bramble) resolveModule(module string) (globals starlark.StringDict, err
 	return b.starlarkExecFile(module, path)
 }
 
-func (b *Bramble) moduleFromPath(path string) (module string, err error) {
-	module = (b.config.Module.Name + "/" + b.relativePathFromConfig())
-	if path == "" {
-		return
-	}
+func (b *Bramble) moduleFromPath(path string) (thisModule string, err error) {
+	thisModule = (b.config.Module.Name + "/" + b.relativePathFromConfig())
 
 	// See if this path is actually the name of a module, for now we just
 	// support one module.
@@ -1327,29 +1350,28 @@ func (b *Bramble) moduleFromPath(path string) (module string, err error) {
 	}
 
 	// if the relative path is nothing, we've already added the slash above
-	if !strings.HasSuffix(module, "/") {
-		module += "/"
+	if !strings.HasSuffix(thisModule, "/") {
+		thisModule += "/"
 	}
 
 	// support things like bar/main.bramble:foo
-	if strings.HasSuffix(path, BrambleExtension) && fileutil.FileExists(path) {
-		return module + path[:len(path)-len(BrambleExtension)], nil
+	if strings.HasSuffix(path, BrambleExtension) && fileutil.FileExists(filepath.Join(b.wd, path)) {
+		return thisModule + path[:len(path)-len(BrambleExtension)], nil
 	}
 
 	fullName := path + BrambleExtension
-	if !fileutil.FileExists(fullName) {
-		if !fileutil.FileExists(path + "/default.bramble") {
+	if !fileutil.FileExists(filepath.Join(b.wd, fullName)) {
+		if !fileutil.FileExists(filepath.Join(b.wd, path+"/default.bramble")) {
 			return "", errors.Errorf("%q: no such file or directory", path)
 		}
 	}
 	// we found it, return
-	module += filepath.Join(path)
-	return
+	thisModule += filepath.Join(path)
+	return strings.TrimSuffix(thisModule, "/"), nil
 }
 
 func (b *Bramble) relativePathFromConfig() string {
-	wd, _ := os.Getwd()
-	relativePath, _ := filepath.Rel(b.configLocation, wd)
+	relativePath, _ := filepath.Rel(b.configLocation, b.wd)
 	if relativePath == "." {
 		// don't add a dot to the path
 		return ""
@@ -1371,5 +1393,6 @@ func (b *Bramble) parseModuleFuncArgument(args []string) (module, function strin
 	}
 	path, function := firstArgument[:lastIndex], firstArgument[lastIndex+1:]
 	module, err = b.moduleFromPath(path)
+	fmt.Println(module, err)
 	return
 }