diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 6de9b47e..80e6fc0e 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,9 +14,5 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout code uses: actions/checkout@v2 - - name: Go test - run: go test ./... - - name: Build some directly - run: make seed_run simple_run - - name: bramblescripts - run: make bramblescripts_to_test + - name: Run all tests + run: make test diff --git a/Makefile b/Makefile index 70e91982..f3e989e6 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,5 @@ -bramblescript_shell: - go run . script - seed_run: go run . run seed:seed @@ -12,18 +9,26 @@ simple_run: seed/linux-x86_64-seed.tar.gz: ./seed/build.sh -test: +test: go_test bramblescripts_to_test seed_run simple_run drv_test + +go_test: go test -v ./... -reptar: +install: + go install + +bramble_tests: install + bramble test ./tests + +docker_reptar: ## Used to compare reptar output to gnutar cd pkg/reptar && docker build -t reptar . \ && docker run -it reptar sh -bramblescripts_to_test: - go install - bramble run pkg/bramblecmd/examples/run:total_bytes_in_folder - bramble run pkg/bramblecmd/examples:main +bramblescripts_to_test: install + bramble run pkg/bramble/cmd-examples:main -drv_test: - go install +drv_test: install bramble test tests/derivation_test.bramble + +starlark_builder: install + bramble run tests/starlark-builder:run_busybox diff --git a/go.mod b/go.mod index 4f596898..7a15aa6e 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,16 @@ go 1.14 require ( github.com/BurntSushi/toml v0.3.1 + github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 + github.com/alecthomas/colour v0.1.0 // indirect + github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c // indirect github.com/davecgh/go-spew v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/mholt/archiver/v3 v3.3.1-0.20200626164424-d44471c49aa7 github.com/mitchellh/cli v1.1.1 github.com/moby/moby v1.13.1 github.com/pkg/errors v0.9.1 + github.com/sergi/go-diff v1.1.0 // indirect github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 go.starlark.net v0.0.0-20200821142938-949cc6f4b097 diff --git a/go.sum b/go.sum index 9ffc18d5..370cf5ba 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,12 @@ code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38 h1:smF2tmSOzy2Mm+0dGI2AIUHY+w0BUc+4tn40djz7+6U= +github.com/alecthomas/assert v0.0.0-20170929043011-405dbfeb8e38/go.mod h1:r7bzyVFMNntcxPZXK3/+KdruV1H5KSlyVY0gc+NgInI= +github.com/alecthomas/colour v0.1.0 h1:nOE9rJm6dsZ66RGWYSFrXw461ZIt9A6+nHgL7FRrDUk= +github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= +github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c h1:MVVbswUlqicyj8P/JljoocA7AyCo62gzD0O7jfvrhtE= +github.com/alecthomas/repr v0.0.0-20200325044227-4184120f674c/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDafo4= github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= @@ -81,11 +87,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= @@ -107,8 +116,11 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IV golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/notes/goals.md b/notes/goals.md new file mode 100644 index 00000000..a30a4da8 --- /dev/null +++ b/notes/goals.md @@ -0,0 +1,12 @@ +TODO + + +Project goals: + - Easy to use and understand + - Provide primitives and tools to create reproducible builds + - First class support for building docker images + - Binary relocation/renaming + - explicit over implicit + - extremely small learning curve for basic use + - limited number of builtins + - extremely easy to publish and use shared libraries/packages diff --git a/pkg/assert/assert.go b/pkg/assert/assert.go index 26c9bafe..cd63a069 100644 --- a/pkg/assert/assert.go +++ b/pkg/assert/assert.go @@ -100,7 +100,7 @@ var ( func LoadAssertModule() (starlark.StringDict, error) { once.Do(func() { predeclared := starlark.StringDict{ - "error": starlark.NewBuiltin("error", error_), + "error": starlark.NewBuiltin("error", Error), "catch": starlark.NewBuiltin("catch", catch), "matches": starlark.NewBuiltin("matches", matches), "module": starlark.NewBuiltin("module", starlarkstruct.MakeModule), @@ -139,7 +139,7 @@ func matches(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, } // error(x) reports an error to the Go test framework. -func error_(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { +func Error(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { if len(args) != 1 { return nil, fmt.Errorf("error: got %d arguments, want 1", len(args)) } diff --git a/pkg/bramble/bramble.go b/pkg/bramble/bramble.go index 588dfd3e..8b41285e 100644 --- a/pkg/bramble/bramble.go +++ b/pkg/bramble/bramble.go @@ -2,17 +2,16 @@ package bramble import ( "fmt" + "io" "io/ioutil" "os" "path/filepath" "strings" + "github.com/davecgh/go-spew/spew" "github.com/pkg/errors" "github.com/maxmcd/bramble/pkg/assert" - "github.com/maxmcd/bramble/pkg/bramblecmd" - "github.com/maxmcd/bramble/pkg/brambleos" - "github.com/maxmcd/bramble/pkg/derivation" "github.com/maxmcd/bramble/pkg/starutil" "github.com/mitchellh/cli" "go.starlark.net/starlark" @@ -80,7 +79,78 @@ type Bramble struct { predeclared starlark.StringDict config Config configLocation string - derivation *derivation.Function + derivationFn *DerivationFunction + cmd *CmdFunction + session *session + + storePath string + bramblePath string + + moduleCache map[string]string + afterDerivation bool + derivationCallCount int + + moduleEntrypoint string + calledFunction string +} + +// implement derivation.Bramble +func (b *Bramble) BramblePath() string { return b.bramblePath } +func (b *Bramble) StorePath() string { return b.storePath } +func (b *Bramble) ModuleCache() map[string]string { return b.moduleCache } +func (b *Bramble) DerivationCallCount() int { return b.derivationCallCount } +func (b *Bramble) RunEntrypoint() (string, string) { return b.moduleEntrypoint, b.calledFunction } +func (b *Bramble) AfterDerivation() { b.afterDerivation = true } +func (b *Bramble) CalledDerivation() error { + b.derivationCallCount++ + if b.afterDerivation { + return errors.New("build context is dirty, can't call derivation after cmd() or other builtins") + } + return nil +} + +func (b *Bramble) CallInlineDerivationFunction(meta functionBuilderMeta, session *session) (err error) { + newBramble := &Bramble{ + // retain from parent + config: b.config, + configLocation: b.configLocation, + storePath: b.storePath, + bramblePath: b.bramblePath, + + // populate for this task + moduleEntrypoint: meta.Module, + calledFunction: meta.Function, + moduleCache: meta.ModuleCache, + derivationCallCount: 0, + session: session, + } + newBramble.thread = &starlark.Thread{Load: newBramble.load} + // this will pass the session to cmd and os + if err = newBramble.initPredeclared(); err != nil { + return + } + newBramble.derivationFn.DerivationCallCount = meta.DerivationCallCount + + spew.Dump(meta.Module) + globals, err := newBramble.resolveModule(meta.Module) + if err != nil { + return + } + + _, intentionalError := starlark.Call( + newBramble.thread, + globals[meta.Function].(*starlark.Function), + nil, nil, + ) + spew.Dump(intentionalError) + fn := intentionalError.(*starlark.EvalError).Unwrap().(ErrFoundBuildContext).Fn + _, err = starlark.Call(newBramble.thread, fn, nil, nil) + return +} + +func (b *Bramble) reset() { + b.moduleCache = map[string]string{} + b.derivationCallCount = 0 } func (b *Bramble) init() (err error) { @@ -88,19 +158,35 @@ func (b *Bramble) init() (err error) { return errors.New("can't initialize Bramble twice") } + b.moduleCache = map[string]string{} + // ensures we have a bramble.toml in the current or parent dir b.config, b.configLocation, err = findConfig() if err != nil { return } + if b.bramblePath, b.storePath, err = ensureBramblePath(); err != nil { + return + } + b.thread = &starlark.Thread{ Name: "main", Load: b.load, } + if b.session, err = newSession("", nil); err != nil { + return + } + + return b.initPredeclared() +} +func (b *Bramble) initPredeclared() (err error) { + if b.derivationFn != nil { + return errors.New("can't init predeclared twice") + } // creates the derivation function and checks we have a valid bramble path and store - b.derivation, err = derivation.NewFunction(b.thread) + b.derivationFn, err = NewDerivationFunction(b) if err != nil { return } @@ -110,13 +196,14 @@ func (b *Bramble) init() (err error) { return } + b.cmd = NewCmdFunction(b.session) + b.predeclared = starlark.StringDict{ - "derivation": b.derivation, - "cmd": bramblecmd.NewFunction(), - "os": brambleos.OS{}, + "derivation": b.derivationFn, + "cmd": b.cmd, + "os": NewOS(b, b.session), "assert": assertGlobals["assert"], } - return } @@ -177,6 +264,63 @@ func (e *testErrorReporter) Error(err error) { } func (e *testErrorReporter) FailNow() bool { return false } +func (b *Bramble) ExecFile(moduleName, filename string) (globals starlark.StringDict, err error) { + storeLocation, ok := b.moduleCache[moduleName] + var f *os.File + if !ok { + f, err = os.Open(filename) + if err != nil { + return + } + hasher := NewHasher() + if _, err = io.Copy(hasher, f); err != nil { + return nil, err + } + storeLocation = filepath.Join(b.StorePath(), hasher.String()+"-star-prog-cache") + } + var mod *starlark.Program + if fileExists(storeLocation) { + var compiledProgram *os.File + compiledProgram, err = os.Open(storeLocation) + if err != nil { + return + } + mod, err = starlark.CompiledProgram(compiledProgram) + if err != nil { + return + } + } else { + if _, err = f.Seek(0, 0); err != nil { + return + } + _, mod, err = starlark.SourceProgram(filename, f, b.predeclared.Has) + if err != nil { + return + } + cachedProgram, err := os.Create(storeLocation) + if err != nil { + return nil, err + } + if err = mod.Write(cachedProgram); err != nil { + return nil, err + } + } + + // TODO: a cleaner implementation should remove these null checks + if f != nil { + if err = f.Close(); err != nil { + return + } + } + if moduleName != "" { + b.moduleCache[moduleName] = storeLocation + } + + g, err := mod.Init(b.thread, b.predeclared) + g.Freeze() + return g, err +} + func (b *Bramble) test(args []string) (err error) { failFast := true if err = b.init(); err != nil { @@ -191,7 +335,9 @@ func (b *Bramble) test(args []string) (err error) { return errors.Wrap(err, "error finding test files") } for _, filename := range testFiles { - globals, err := starlark.ExecFile(b.thread, filename, nil, b.predeclared) + // TODO: need to calculate module name + b.reset() + globals, err := b.ExecFile("", filename) if err != nil { return err } @@ -253,7 +399,7 @@ func (b *Bramble) resolveModule(module string) (globals starlark.StringDict, err return } - return starlark.ExecFile(b.thread, path, nil, b.predeclared) + return b.ExecFile(module, path) } func (b *Bramble) run(args []string) (err error) { @@ -274,6 +420,8 @@ func (b *Bramble) run(args []string) (err error) { if err != nil { return } + b.calledFunction = fn + b.moduleEntrypoint = module toCall, ok := globals[fn] if !ok { return errors.Errorf("global function %q not found", fn) @@ -289,7 +437,11 @@ func (b *Bramble) moduleFromPath(path string) (module string, err error) { if path == "" { return } - module += "/" + + // if the relative path is nothing, we've already added the slash above + if !strings.HasSuffix(module, "/") { + module += "/" + } // support things like bar/main.bramble:foo if strings.HasSuffix(path, extension) && fileExists(path) { @@ -310,6 +462,10 @@ func (b *Bramble) moduleFromPath(path string) (module string, err error) { func (b *Bramble) relativePathFromConfig() string { wd, _ := os.Getwd() relativePath, _ := filepath.Rel(b.configLocation, wd) + if relativePath == "." { + // don't add a dot to the path + return "" + } return relativePath } diff --git a/pkg/bramblecmd/examples.bramble b/pkg/bramble/cmd-examples.bramble similarity index 98% rename from pkg/bramblecmd/examples.bramble rename to pkg/bramble/cmd-examples.bramble index b8e86c2d..18b8c6fc 100755 --- a/pkg/bramblecmd/examples.bramble +++ b/pkg/bramble/cmd-examples.bramble @@ -1,5 +1,5 @@ """ -Bramblescript examples +Bramble cmd examples """ diff --git a/pkg/bramblecmd/cmd.go b/pkg/bramble/cmd.go similarity index 52% rename from pkg/bramblecmd/cmd.go rename to pkg/bramble/cmd.go index fd5989b6..62f3a26f 100644 --- a/pkg/bramblecmd/cmd.go +++ b/pkg/bramble/cmd.go @@ -1,22 +1,190 @@ -package bramblecmd +package bramble import ( + "bufio" "bytes" "fmt" + "io" + "io/ioutil" "os/exec" "path/filepath" "strings" "sync" + "syscall" "github.com/kballard/go-shellquote" "github.com/maxmcd/bramble/pkg/starutil" + "github.com/moby/moby/pkg/stdcopy" "github.com/pkg/errors" "go.starlark.net/starlark" ) +var ErrInvalidRead = errors.New("can't read from command output more than once") + +// CmdFunction is the value for the builtin "cmd", calling it as a function +// creates a new cmd instance, it also has various other attributes and methods +type CmdFunction struct { + session *session +} + +var ( + _ starlark.Value = new(CmdFunction) + _ starlark.Callable = new(CmdFunction) +) + +func NewCmdFunction(session *session) *CmdFunction { + return &CmdFunction{session: session} +} + +func (fn *CmdFunction) Freeze() {} +func (fn *CmdFunction) Hash() (uint32, error) { return 0, starutil.ErrUnhashable(fn.Type()) } +func (fn *CmdFunction) Name() string { return fn.Type() } +func (fn *CmdFunction) String() string { return "" } +func (fn *CmdFunction) Type() string { return "builtin_function_cmd" } +func (fn *CmdFunction) Truth() starlark.Bool { return true } + +// CallInternal defines the cmd() starlark function. +func (fn *CmdFunction) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (v starlark.Value, err error) { + return fn.newCmd(thread, args, kwargs, nil, "") +} + +// NewCmd creates a new cmd instance given args and kwargs. NewCmd will error +// immediately if it can't find the cmd +func (fn *CmdFunction) newCmd(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple, stdin *Cmd, dir string) (val starlark.Value, err error) { + cmd := Cmd{fn: fn} + cmd.Dir = fn.session.currentDirectory + + var stdinKwarg starlark.Value + var dirKwarg starlark.String + var envKwarg *starlark.Dict + var clearEnvKwarg starlark.Bool + var ignoreFailureKwarg starlark.Bool + var printOutputKwarg starlark.Bool + if err = starlark.UnpackArgs("f", nil, kwargs, + "stdin?", &stdinKwarg, + "dir?", &dirKwarg, + "env?", &envKwarg, + "clear_env?", &clearEnvKwarg, + "ignore_failure?", &ignoreFailureKwarg, + "print_output?", &printOutputKwarg, + ); err != nil { + return + } + + if clearEnvKwarg == starlark.True { + cmd.Env = []string{} + } else { + cmd.Env = fn.session.envArray() + } + if envKwarg != nil { + kvs, err := starutil.DictToGoStringMap(envKwarg) + if err != nil { + return nil, err + } + for k, v := range kvs { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + } + + // and empty cmd() call isn't allowed + if args.Len() == 0 { + return nil, errors.New("cmd() missing 1 required positional argument") + } + + if args.Index(0).Type() == "function" { + fn := args.Index(0).(callInternalable) + kwargs = append(kwargs, starlark.Tuple{starlark.String("stdin"), stdin}) + return fn.CallInternal(thread, args[1:], kwargs) + } + + // it's cmd(["grep", "-v"]) + if args.Len() == 1 { + if args.Index(0).Type() == "list" { + cmd.Args, err = starutil.ListToListOfStrings(args.Index(0)) + if err != nil { + return nil, err + } + if len(cmd.Args) == 0 { + return nil, errors.New("if the first argument is a list it can't be empty") + } + } else if args.Index(0).Type() == "string" { + starlarkCmd := args.Index(0).(starlark.String).GoString() + if starlarkCmd == "" { + return nil, errors.New("if the first argument is a string it can't be empty") + } + cmd.Args, err = shellquote.Split(starlarkCmd) + if err != nil { + return + } + if len(cmd.Args) == 0 { + // whitespace bash characters will be removed by shellquote, + // add them back for correct error message + cmd.Args = []string{starlarkCmd} + } + } + } else { + iterator := args.Iterate() + defer iterator.Done() + var val starlark.Value + for iterator.Next(&val) { + if err := cmd.addArgumentToCmd(val); err != nil { + return nil, err + } + } + } + + // expand shell variables in arguments + cmd.expandArguments() + + if stdinKwarg != nil { + if err = cmdAttachStdin(&cmd, stdinKwarg); err != nil { + return + } + } + // kwargs: + // stdin + // dir + // env + name := cmd.name() + if filepath.Base(name) == name { + var lp string + if lp, err = exec.LookPath(name); err != nil { + return nil, err + } + cmd.Path = lp + } else { + cmd.Path = name + } + + cmd.wg = &sync.WaitGroup{} + cmd.wg.Add(1) + + cmd.ss, cmd.Stdout, cmd.Stderr = NewStandardStream() + if stdin != nil { + cmd.Stdin = stdin + } + err = cmd.Start() + go func() { + err := cmd.Cmd.Wait() + { + cmd.lock.Lock() + if err != nil { + cmd.err = err + } + cmd.finished = true + cmd.lock.Unlock() + } + cmd.ss.Close() + cmd.wg.Done() + }() + return &cmd, err +} + type Cmd struct { exec.Cmd + + fn *CmdFunction frozen bool finished bool err error @@ -120,12 +288,12 @@ func (cmd *Cmd) IfErr(thread *starlark.Thread, args starlark.Tuple, kwargs []sta } // if there is an error we run the command in or instead - return newCmd(thread, args, kwargs, nil, cmd.Dir) + return cmd.fn.newCmd(thread, args, kwargs, nil, cmd.Dir) } func (cmd *Cmd) Pipe(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { _ = cmd.setOutput(true, false) - return newCmd(thread, args, kwargs, cmd, cmd.Dir) + return cmd.fn.newCmd(thread, args, kwargs, cmd, cmd.Dir) } func (cmd *Cmd) addArgumentToCmd(value starlark.Value) (err error) { @@ -137,6 +305,12 @@ func (cmd *Cmd) addArgumentToCmd(value starlark.Value) (err error) { return } +func (cmd *Cmd) expandArguments() { + for i, arg := range cmd.Args { + cmd.Args[i] = cmd.fn.session.expand(arg) + } +} + func (cmd *Cmd) starlarkWait(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { return cmd, cmd.Wait() } @@ -163,7 +337,7 @@ type callInternalable interface { CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (v starlark.Value, err error) } -func attachStdin(cmd *Cmd, val starlark.Value) (err error) { +func cmdAttachStdin(cmd *Cmd, val starlark.Value) (err error) { switch v := val.(type) { case *Cmd: if err = v.setOutput(true, false); err != nil { @@ -183,159 +357,231 @@ func attachStdin(cmd *Cmd, val starlark.Value) (err error) { return nil } -// NewCmd creates a new cmd instance given args and kwargs. NewCmd will error -// immediately if it can't find the cmd -func newCmd(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple, stdin *Cmd, dir string) (val starlark.Value, err error) { - fmt.Println(args) - // if input is an array we use the first item as the cmd - // if input is just args we use them as cmd+args - // if input is just a string we parse it as a shell command +func (cmd *Cmd) setOutput(stdout, stderr bool) (err error) { + if cmd.ss.started { + return ErrInvalidRead + } + cmd.ss.stdout = stdout + cmd.ss.stderr = stderr + return +} - cmd := Cmd{} - cmd.Dir = dir +func (cmd *Cmd) Read(p []byte) (n int, err error) { + n, err = cmd.ss.Read(p) + cmd.lock.Lock() + defer cmd.lock.Unlock() + if cmd.err != nil { + return n, cmd.err + } - var stdinKwarg starlark.Value - var dirKwarg starlark.String - var envKwarg *starlark.Dict - var clearEnvKwarg starlark.Bool - var ignoreFailureKwarg starlark.Bool - var printOutputKwarg starlark.Bool - if err = starlark.UnpackArgs("f", nil, kwargs, - "stdin?", &stdinKwarg, - "dir?", &dirKwarg, - "env?", &envKwarg, - "clear_env?", &clearEnvKwarg, - "ignore_failure?", &ignoreFailureKwarg, - "print_output?", &printOutputKwarg, - ); err != nil { + return +} + +type ByteStream struct { + cmd *Cmd + stdout bool + stderr bool +} + +var ( + _ starlark.Value = ByteStream{} + _ starlark.Callable = ByteStream{} + _ starlark.Iterable = ByteStream{} +) + +func (bs ByteStream) Name() string { + if bs.stdout && bs.stderr { + return "output" + } + if bs.stderr { + return "stderr" + } + return "stdout" +} +func (bs ByteStream) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + if err = bs.cmd.setOutput(bs.stdout, bs.stderr); err != nil { return } - - if clearEnvKwarg == starlark.True { - cmd.Env = []string{} + b, err := ioutil.ReadAll(bs.cmd) + if err == io.ErrClosedPipe { + err = nil } - if envKwarg != nil { - for _, key := range envKwarg.Keys() { - envVal, _, _ := envKwarg.Get(key) - keyString, err := starutil.ValueToString(key) - if err != nil { - return nil, err - } - valString, err := starutil.ValueToString(envVal) - if err != nil { - return nil, err - } - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", keyString, valString)) - } + if err != nil { + // TODO: need a better solution for this + fmt.Println(string(b)) } + return starlark.String(string(b)), err +} - // cmd() isn't allowed - if args.Len() == 0 { - return nil, errors.New("cmd() missing 1 required positional argument") +func (bs ByteStream) String() string { + return fmt.Sprintf("", bs.Name()) +} + +func (bs ByteStream) Type() string { return "bytestream" } +func (bs ByteStream) Freeze() {} +func (bs ByteStream) Truth() starlark.Bool { return bs.cmd.Truth() } +func (bs ByteStream) Hash() (uint32, error) { return 0, errors.New("bytestream is unhashable") } + +func (bs ByteStream) Iterate() starlark.Iterator { + err := bs.cmd.setOutput(bs.stdout, bs.stderr) + bsi := byteStreamIterator{ + bs: bs, + } + if err == nil { + bsi.buf = bufio.NewReader(bs.cmd) } + return bsi +} - if args.Index(0).Type() == "function" { - fn := args.Index(0).(callInternalable) - kwargs = append(kwargs, starlark.Tuple{starlark.String("stdin"), stdin}) - return fn.CallInternal(thread, args[1:], kwargs) +type byteStreamIterator struct { + bs ByteStream + buf *bufio.Reader +} + +func (bsi byteStreamIterator) Next(p *starlark.Value) bool { + if bsi.buf == nil { + return false + } + str, err := bsi.buf.ReadString('\n') + if err == io.EOF || err == io.ErrClosedPipe { + return false } + if err != nil { + // TODO: something better here? certain errors we care about? + panic(err) + } + *p = starlark.String(str[:len(str)-1]) + return true +} - // it's cmd(["grep", "-v"]) - if args.Len() == 1 { - if args.Index(0).Type() == "list" { - cmd.Args, err = starutil.ListToListOfStrings(args.Index(0)) - if err != nil { - return nil, err - } - if len(cmd.Args) == 0 { - return nil, errors.New("if the first argument is a list it can't be empty") - } - } else if args.Index(0).Type() == "string" { - starlarkCmd := args.Index(0).(starlark.String).GoString() - if starlarkCmd == "" { - return nil, errors.New("if the first argument is a string it can't be empty") - } - cmd.Args, err = shellquote.Split(starlarkCmd) - if err != nil { - return - } - if len(cmd.Args) == 0 { - // whitespace bash characters will be removed by shellquote, - // add them back for correct error message - cmd.Args = []string{starlarkCmd} - } +func (bsi byteStreamIterator) Done() {} + +// bufferedPipe is a buffered pipe +// parts taken from https://github.com/golang/go/blob/0436b162397018c45068b47ca1b5924a3eafdee0/src/net/net_fake.go#L173 +type bufferedPipe struct { + softLimit int + mu sync.Mutex + buf []byte + closed bool + rCond sync.Cond + wCond sync.Cond +} + +func newBufferedPipe(softLimit int) *bufferedPipe { + p := &bufferedPipe{softLimit: softLimit} + p.rCond.L = &p.mu + p.wCond.L = &p.mu + return p +} + +func (p *bufferedPipe) Read(b []byte) (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + for { + if p.closed && len(p.buf) == 0 { + return 0, io.EOF } - } else { - iterator := args.Iterate() - defer iterator.Done() - var val starlark.Value - for iterator.Next(&val) { - if err := cmd.addArgumentToCmd(val); err != nil { - return nil, err - } + if len(p.buf) > 0 { + break } + p.rCond.Wait() } - if stdinKwarg != nil { - if err = attachStdin(&cmd, stdinKwarg); err != nil { - return + n := copy(b, p.buf) + p.buf = p.buf[n:] + p.wCond.Broadcast() + return n, nil +} + +func (p *bufferedPipe) Write(b []byte) (int, error) { + p.mu.Lock() + defer p.mu.Unlock() + + for { + if p.closed { + return 0, syscall.ENOTCONN } - } - // kwargs: - // stdin - // dir - // env - name := cmd.name() - if filepath.Base(name) == name { - var lp string - if lp, err = exec.LookPath(name); err != nil { - return nil, err + if len(p.buf) <= p.softLimit { + break } - cmd.Path = lp + p.wCond.Wait() } - cmd.wg = &sync.WaitGroup{} - cmd.wg.Add(1) + p.buf = append(p.buf, b...) + p.rCond.Broadcast() + return len(b), nil +} - cmd.ss, cmd.Stdout, cmd.Stderr = NewStandardStream() - if stdin != nil { - cmd.Stdin = stdin - } - err = cmd.Start() - logger.Println(cmd.String(), "started") - go func() { - err := cmd.Cmd.Wait() - { - cmd.lock.Lock() - if err != nil { - logger.Println(cmd.String(), "error", err) - cmd.err = err - } - cmd.finished = true - cmd.lock.Unlock() - } - cmd.ss.Close() - cmd.wg.Done() - }() - return &cmd, err +func (p *bufferedPipe) Close() (err error) { + p.mu.Lock() + defer p.mu.Unlock() + + p.closed = true + p.rCond.Broadcast() + p.wCond.Broadcast() + return nil } -func (cmd *Cmd) setOutput(stdout, stderr bool) (err error) { - if cmd.ss.started { - return ErrInvalidRead +type StandardStream struct { + stdout bool + stderr bool + once sync.Once + + buffPipe *bufferedPipe + + reader *io.PipeReader + + err error + lock sync.Mutex + + started bool +} + +func NewStandardStream() (ss *StandardStream, stdoutWriter, stderrWriter io.Writer) { + ss = &StandardStream{ + buffPipe: newBufferedPipe(4096 * 16), } - cmd.ss.stdout = stdout - cmd.ss.stderr = stderr + + stdoutWriter = stdcopy.NewStdWriter(ss.buffPipe, stdcopy.Stdout) + stderrWriter = stdcopy.NewStdWriter(ss.buffPipe, stdcopy.Stderr) + return } -func (cmd *Cmd) Read(p []byte) (n int, err error) { - n, err = cmd.ss.Read(p) - cmd.lock.Lock() - defer cmd.lock.Unlock() - if cmd.err != nil { - return n, cmd.err - } +func (ss *StandardStream) Close() (err error) { + _ = ss.buffPipe.Close() + return nil +} +func (ss *StandardStream) Read(p []byte) (n int, err error) { + ss.once.Do(func() { + ss.started = true + var writer *io.PipeWriter + ss.lock.Lock() + ss.reader, writer = io.Pipe() + ss.lock.Unlock() + var stdoutWriter io.Writer = ioutil.Discard + var stderrWriter io.Writer = ioutil.Discard + if ss.stdout { + stdoutWriter = writer + } + if ss.stderr { + stderrWriter = writer + } + go func() { + _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, ss.buffPipe) + ss.lock.Lock() + ss.err = err + ss.lock.Unlock() + _ = writer.Close() + _ = ss.reader.Close() + }() + }) + n, err = ss.reader.Read(p) + ss.lock.Lock() + if err == nil && ss.err != nil { + err = ss.err + } + ss.lock.Unlock() return } diff --git a/pkg/bramble/cmd_test.go b/pkg/bramble/cmd_test.go new file mode 100644 index 00000000..f1166e04 --- /dev/null +++ b/pkg/bramble/cmd_test.go @@ -0,0 +1,192 @@ +package bramble + +import ( + "testing" + + "github.com/alecthomas/assert" + "go.starlark.net/starlark" +) + +type cmdTest struct { + name string + script string + errContains string + returnValue string +} + +func runCmdTest(t *testing.T, tests []cmdTest) { + session, err := newSession("", nil) + if err != nil { + t.Error(err) + } + for _, tt := range tests { + t.Run(tt.script, func(t *testing.T) { + thread := &starlark.Thread{Name: "main"} + cmd := NewCmdFunction(session) + globals, err := starlark.ExecFile( + thread, tt.name+".bramble", + tt.script, starlark.StringDict{"cmd": cmd}, + ) + if err != nil || tt.errContains != "" { + if err == nil { + t.Error("error is nil") + return + } + assert.Contains(t, err.Error(), tt.errContains) + if tt.errContains == "" { + t.Error(err, tt.script) + return + } + } + if tt.returnValue == "" { + return + } + b, ok := globals["b"] + if !ok { + t.Errorf("%q doesn't output global value b", tt.script) + return + } + assert.Equal(t, tt.returnValue, b.String()) + }) + } +} + +func TestStarlarkCmd(t *testing.T) { + tests := []cmdTest{ + {script: ` +c = cmd("ls") +b = [getattr(c, x) for x in dir(c)] + `, + returnValue: ""}, + {script: "cmd()", + errContains: "missing 1 required positional argument"}, + {script: "cmd([])", + errContains: "be empty"}, + {script: `cmd("")`, + errContains: "be empty"}, + {script: `cmd(" ")`, + errContains: `" "`}, + {script: `b=[getattr(cmd("echo"), x) for x in dir(cmd("echo"))]`, + returnValue: ``}, + {script: `cmd("sleep 2").kill()`}, + {script: `b=cmd(["ls", "-lah"])`, + returnValue: ``}, + {script: `b=cmd("ls -lah")`, + returnValue: ``}, + {script: `b=cmd("ls \"-lah\"")`, + returnValue: ``}, + {script: `b=cmd("ls '-lah'")`, + returnValue: ``}, + {script: `b=cmd("ls", "-lah")`, + returnValue: ``}, + {script: `b=cmd("echo 'these are words'").pipe("tr ' ' '\n'").pipe("grep these").stdout()`, + returnValue: `"these\n"`}, + {script: `b=cmd("ls -lah").wait().exit_code`, + returnValue: `0`}, + {script: `c = cmd("echo") +cmd("echo").wait() +c.kill()`}, + } + runCmdTest(t, tests) +} + +func TestPipe(t *testing.T) { + tests := []cmdTest{ + {script: `b=cmd("echo 'these are words'").pipe("tr ' ' '\n'").pipe("grep these").stdout()`, + returnValue: `"these\n"`}, + } + runCmdTest(t, tests) +} + +func TestCallback(t *testing.T) { + tests := []cmdTest{ + {script: ` +def echo(*args, **kwargs): + return cmd("echo", *args, **kwargs) + +b=echo("hi").stdout().strip() +`, + returnValue: `"hi"`}, + } + runCmdTest(t, tests) +} + +func TestArgs(t *testing.T) { + tests := []cmdTest{ + {script: `b=cmd("grep hi", stdin=cmd("echo hi")).output()`, + returnValue: `"hi\n"`}, + {script: `b=cmd("grep hi", stdin="hi").output()`, + returnValue: `"hi\n"`}, + {script: `b=cmd("env", clear_env=True).output()`, + returnValue: `""`}, + {script: `b=cmd("env", clear_env=True, env={"foo":"bar", "baz": 1}).output()`, + returnValue: `"foo=bar\nbaz=1\n"`}, + } + runCmdTest(t, tests) +} + +func TestIfErr(t *testing.T) { + tests := []cmdTest{ + {script: `b=cmd("ls", "notathing").if_err("echo", "hi").stdout()`, + returnValue: `"hi\n"`}, + {script: `b=cmd("ls", "notathing").if_err("echo", "hi").stdout()`, + returnValue: `"hi\n"`}, + } + runCmdTest(t, tests) +} + +func TestCallable(t *testing.T) { + tests := []cmdTest{ + {script: `b=cmd("ls").pipe`, + returnValue: ``}, + {script: `b=type(cmd("ls").if_err)`, + returnValue: `"builtin_function_or_method"`}, + } + runCmdTest(t, tests) +} + +func TestByteStream(t *testing.T) { + tests := []cmdTest{ + {script: `b=cmd("ls", "notathing").stdout()`, + errContains: `exit`}, + {script: `b=cmd("echo","hi").stdout()`, + returnValue: `"hi\n"`}, + {script: `b=list(cmd("echo","hi").stdout)`, + returnValue: `["hi"]`}, + {script: `b=cmd("echo","hi").stdout`, + returnValue: ``}, + {script: `b=type(cmd("echo","hi").stdout)`, + returnValue: `"bytestream"`}, + {script: `b=cmd("echo", "hi").output()`, + returnValue: `"hi\n"`}, + {script: `b=cmd("echo", "hi").stderr()`, + returnValue: `""`}, + {script: `b=cmd("echo", "hi").stderr`, + returnValue: ``}, + {script: `b=cmd("echo", "hi").stdout`, + returnValue: ``}, + {script: `b=cmd("echo", "hi").output`, + returnValue: ``}, + {script: `b=cmd("echo", 1).stdout()`, + returnValue: `"1\n"`}, + {script: ` +def run(): + c = cmd("ls") + response = "" + for line in c.stdout: + if "cmd_test" in line: + response = line + return response +b = run()`, + returnValue: `"cmd_test.go"`}, + {script: ` +def run(): + c = cmd("ls") + for line in c.output: + if "cmd_test" in line: + return line +b = run()`, + returnValue: `"cmd_test.go"`}, + } + runCmdTest(t, tests) +} diff --git a/pkg/bramble/derivation.go b/pkg/bramble/derivation.go new file mode 100644 index 00000000..44a84217 --- /dev/null +++ b/pkg/bramble/derivation.go @@ -0,0 +1,882 @@ +package bramble + +import ( + "bytes" + "crypto/sha256" + "encoding/base32" + "encoding/hex" + "encoding/json" + "fmt" + "hash" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + "syscall" + + "github.com/davecgh/go-spew/spew" + "github.com/maxmcd/bramble/pkg/reptar" + "github.com/maxmcd/bramble/pkg/starutil" + "github.com/maxmcd/bramble/pkg/textreplace" + "github.com/mholt/archiver/v3" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "go.starlark.net/resolve" + "go.starlark.net/starlark" +) + +var ( + TempDirPrefix = "bramble-" + + // BramblePrefixOfRecord is the prefix we use when hashing the build output + // this allows us to get a consistent hash even if we're building in a + // different location + BramblePrefixOfRecord = "/home/bramble/bramble/bramble_store_padding/bramb" +) + +// DerivationFunction is the function that creates derivations +type DerivationFunction struct { + bramble *Bramble + + derivations map[string]*Derivation + + log *logrus.Logger + + DerivationCallCount int +} + +var ( + _ starlark.Value = new(DerivationFunction) + _ starlark.Callable = new(DerivationFunction) +) + +func (f *DerivationFunction) Freeze() {} +func (f *DerivationFunction) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("module") } +func (f *DerivationFunction) Name() string { return f.String() } +func (f *DerivationFunction) String() string { return `` } +func (f *DerivationFunction) Truth() starlark.Bool { return true } +func (f *DerivationFunction) Type() string { return "module" } + +func init() { + // It's easier to start giving away free coffee than it is to take away + // free coffee + resolve.AllowFloat = false + resolve.AllowLambda = false + resolve.AllowNestedDef = false + resolve.AllowRecursion = false + resolve.AllowSet = true +} + +// NewDerivationFunction creates a new client. When initialized this function checks if the +// bramble store exists and creates it if it does not. +func NewDerivationFunction(bramble *Bramble) (*DerivationFunction, error) { + fn := &DerivationFunction{ + log: logrus.New(), + derivations: make(map[string]*Derivation), + bramble: bramble, + } + // c.log.SetReportCaller(true) + fn.log.SetLevel(logrus.DebugLevel) + + return fn, nil +} + +func (f *DerivationFunction) joinStorePath(v ...string) string { + return filepath.Join(append([]string{f.bramble.StorePath()}, v...)...) +} + +// Load derivation will load and parse a derivation from the bramble store1 +func (f *DerivationFunction) LoadDerivation(filename string) (drv *Derivation, exists bool, err error) { + fileLocation := f.joinStorePath(filename) + _, err = os.Stat(fileLocation) + if err != nil { + return nil, false, nil + } + file, err := os.Open(fileLocation) + if err != nil { + return nil, true, err + } + drv = &Derivation{} + return drv, true, json.NewDecoder(file).Decode(drv) +} + +func (f *DerivationFunction) buildDerivation(drv *Derivation) (err error) { + var exists bool + exists, err = drv.checkForExisting() + if err != nil { + return + } + if exists { + return + } + if err = drv.build(); err != nil { + return + } + if err = drv.writeDerivation(); err != nil { + return + } + return +} + +// DownloadFile downloads a file into the store. Must include an expected hash +// of the downloaded file as a hex string of a sha256 hash +func (f *DerivationFunction) DownloadFile(url string, hash string) (path string, err error) { + f.log.Debugf("Downloading url %s", url) + + b, err := hex.DecodeString(hash) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("error decoding hash %q; is it hexadecimal?", hash)) + return + } + storePrefixHash := bytesToBase32Hash(b) + matches, err := filepath.Glob(f.joinStorePath(storePrefixHash) + "*") + if err != nil { + err = errors.Wrap(err, "error searching for existing hashed content") + return + } + if len(matches) != 0 { + return matches[0], nil + } + resp, err := http.Get(url) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("error making request to download %q", url)) + return + } + defer resp.Body.Close() + file, err := ioutil.TempFile(filepath.Join(f.bramble.BramblePath(), "tmp"), "") + if err != nil { + err = errors.Wrap(err, "error creating a temporary file for a download") + return + } + sha256Hash := sha256.New() + tee := io.TeeReader(resp.Body, sha256Hash) + if _, err = io.Copy(file, tee); err != nil { + err = errors.Wrap(err, "error writing to the temporary download file") + return + } + sha256HashBytes := sha256Hash.Sum(nil) + hexStringHash := fmt.Sprintf("%x", sha256HashBytes) + if hash != hexStringHash { + err = errors.Errorf( + "Got incorrect hash for url %s.\nwanted %q\ngot %q", + url, hash, hexStringHash) + // make best effort to save this file, as we'll likely just download it again + storePrefixHash = bytesToBase32Hash(sha256HashBytes) + } + path = f.joinStorePath(storePrefixHash + "-" + filepath.Base(url)) + // don't overwrite err if we error here, we want to try and save this, but + // still return the incorrect hash error + if er := os.Rename(file.Name(), path); er != nil { + return "", errors.Wrap(er, "error moving file into store") + } + return path, err +} + +type ErrFoundBuildContext struct { + thread *starlark.Thread + Fn *starlark.Function +} + +func (efbc ErrFoundBuildContext) Error() string { + return "internal err found build context error" +} + +func getBuilderFunction(kwargs []starlark.Tuple) *starlark.Function { + for _, tup := range kwargs { + key, val := tup[0].(starlark.String), tup[1] + if key == "builder" { + if fn, ok := val.(*starlark.Function); ok { + return fn + } + } + } + return nil +} + +func (f *DerivationFunction) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (v starlark.Value, err error) { + if err = f.bramble.CalledDerivation(); err != nil { + return + } + + // we're running inside a derivation build and we need to exit with this function and function context + if f.bramble.DerivationCallCount() == f.DerivationCallCount { + return nil, ErrFoundBuildContext{thread: thread, Fn: getBuilderFunction(kwargs)} + } + if args.Len() > 0 { + return nil, errors.New("builtin function build() takes no positional arguments") + } + drv, err := f.newDerivationFromArgs(args, kwargs) + if err != nil { + return nil, err + } + // At(0) is within this function, we want the file of the caller + drv.location = filepath.Dir(thread.CallStack().At(1).Pos.Filename()) + if err = drv.calculateInputDerivations(); err != nil { + return nil, err + } + f.log.Debug("Calculated derivation before build: ", drv.prettyJSON()) + + f.log.Debugf("Building derivation %q", drv.Name) + if err = f.buildDerivation(drv); err != nil { + return nil, err + } + f.log.Debug("Completed derivation: ", drv.prettyJSON()) + _, filename, err := drv.computeDerivation() + if err != nil { + return + } + f.derivations[filename] = drv + return drv, nil +} + +type functionBuilderMeta struct { + DerivationCallCount int + ModuleCache map[string]string + Module string + Function string +} + +// setBuilder is used during instantiation to set various attributes on the +// derivation for a specific builder +func setBuilder(drv *Derivation, builder starlark.Value) (err error) { + switch v := builder.(type) { + case starlark.String: + drv.Builder = v.GoString() + case *starlark.Function: + drv.Builder = "function" + meta := functionBuilderMeta{ + DerivationCallCount: drv.function.bramble.DerivationCallCount(), + ModuleCache: drv.function.bramble.ModuleCache(), + } + meta.Module, meta.Function = drv.function.bramble.RunEntrypoint() + + b, _ := json.Marshal(meta) + drv.Env["function_builder_meta"] = string(b) + default: + return errors.Errorf("no builder for %q", builder.Type()) + } + return +} + +func (f *DerivationFunction) newDerivationFromArgs(args starlark.Tuple, kwargs []starlark.Tuple) (drv *Derivation, err error) { + drv = &Derivation{ + Outputs: map[string]Output{"out": {}}, + Env: map[string]string{}, + function: f, + } + var ( + name starlark.String + builder starlark.Value = starlark.None + argsParam *starlark.List + sources *starlark.List + env *starlark.Dict + outputs *starlark.List + buildInputs *starlark.List + ) + if err = starlark.UnpackArgs("derivation", args, kwargs, + "builder", &builder, + "name?", &name, + "args?", &argsParam, + "sources?", &sources, + "env?", &env, + "outputs?", &outputs, + "build_inputs", &buildInputs, + ); err != nil { + return + } + + drv.Name = name.GoString() + + if argsParam != nil { + if drv.Args, err = starutil.ListToGoList(argsParam); err != nil { + return + } + } + if sources != nil { + if drv.Sources, err = starutil.ListToGoList(sources); err != nil { + return + } + } + if env != nil { + if drv.Env, err = starutil.DictToGoStringMap(env); err != nil { + return + } + } + + if buildInputs != nil { + for _, item := range starutil.ListToValueList(buildInputs) { + input, ok := item.(*Derivation) + if !ok { + err = errors.Errorf("build_inputs takes a list of derivations, found type %q", item.Type()) + return + } + _, filename, err := input.computeDerivation() + if err != nil { + panic(err) + } + drv.InputDerivations = append(drv.InputDerivations, InputDerivation{ + Path: filename, + Output: "out", // TODO: support passing other outputs + }) + drv.Env[input.Name] = "$bramble_path/" + input.Outputs["out"].Path + } + } + if outputs != nil { + outputsList, err := starutil.ListToGoList(outputs) + if err != nil { + return nil, err + } + delete(drv.Outputs, "out") + for _, o := range outputsList { + drv.Outputs[o] = Output{} + } + } + + if err = setBuilder(drv, builder); err != nil { + return + } + + return drv, nil +} + +// Derivation is the basic building block of a Bramble build +type Derivation struct { + Name string + Outputs map[string]Output + Builder string + Platform string + Args []string + Env map[string]string + Sources []string + InputDerivations []InputDerivation + + // internal fields + function *DerivationFunction + location string +} + +// DerivationOutput tracks the build outputs. Outputs are not included in the +// Derivation hash. The path tracks the output location in the bramble store +// and Dependencies tracks the bramble outputs that are runtime dependencies. +type Output struct { + Path string + Dependencies []string +} + +// InputDerivation is one of the derivation inputs. Path is the location of +// the derivation, output is the name of the specific output this derivation +// uses for the build +type InputDerivation struct { + Path string + Output string +} + +var ( + _ starlark.Value = new(Derivation) + _ starlark.HasAttrs = new(Derivation) +) + +func (drv *Derivation) Freeze() {} +func (drv *Derivation) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("cmd") } +func (drv *Derivation) String() string { return fmt.Sprintf("", drv.Name) } +func (drv *Derivation) Truth() starlark.Bool { return starlark.True } +func (drv *Derivation) Type() string { return "derivation" } + +func (drv *Derivation) Attr(name string) (val starlark.Value, err error) { + output, ok := drv.Outputs[name] + if ok { + return starlark.String(fmt.Sprintf("$bramble_path/%s", output.Path)), nil + } + return nil, nil +} + +func (drv *Derivation) AttrNames() (out []string) { + for name := range drv.Outputs { + out = append(out, name) + } + return +} + +func (drv *Derivation) prettyJSON() string { + b, _ := json.MarshalIndent(drv, "", " ") + return string(b) +} + +func (drv *Derivation) calculateInputDerivations() (err error) { + // TODO: is this the best way to do this? presumaby in nix it's a language + // feature + + fileBytes, err := json.Marshal(drv) + if err != nil { + return + } + for location, derivation := range drv.function.derivations { + // TODO: check all outputs, not just the default + if bytes.Contains(fileBytes, []byte(derivation.String())) { + drv.InputDerivations = append(drv.InputDerivations, InputDerivation{ + Path: location, + Output: "out", + }) + } + } + sort.Slice(drv.InputDerivations, func(i, j int) bool { + id := drv.InputDerivations[i] + jd := drv.InputDerivations[j] + return id.Path+id.Output < jd.Path+id.Output + }) + return nil +} + +func (drv *Derivation) computeDerivation() (fileBytes []byte, filename string, err error) { + fileBytes, err = json.Marshal(drv) + if err != nil { + return + } + outputs := drv.Outputs + // content is hashed without the outputs attribute + drv.Outputs = nil + var jsonBytesForHashing []byte + jsonBytesForHashing, err = json.Marshal(drv) + if err != nil { + return + } + drv.Outputs = outputs + fileName := fmt.Sprintf("%s.drv", drv.Name) + _, filename, err = hashFile(fileName, ioutil.NopCloser(bytes.NewBuffer(jsonBytesForHashing))) + if err != nil { + return + } + return +} + +func (drv *Derivation) checkForExisting() (exists bool, err error) { + _, filename, err := drv.computeDerivation() + if err != nil { + return + } + drv.function.log.Debug("derivation " + drv.Name + " evaluates to " + filename) + existingDrv, exists, err := drv.function.LoadDerivation(filename) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + for _, v := range existingDrv.Outputs { + if v.Path == "" { + return false, nil + } + } + drv.Outputs = existingDrv.Outputs + return true, nil +} + +func (drv *Derivation) assembleSources(destination string) (runLocation string, err error) { + if len(drv.Sources) == 0 { + return + } + sources := drv.Sources + drv.Sources = []string{} + absDir, err := filepath.Abs(drv.location) + if err != nil { + return + } + // get absolute paths for all sources + for i, src := range sources { + sources[i] = filepath.Join(absDir, src) + } + prefix := commonFilepathPrefix(append(sources, absDir)) + + if err = copyFiles(prefix, sources, destination); err != nil { + return + } + relBramblefileLocation, err := filepath.Rel(prefix, absDir) + if err != nil { + return "", errors.Wrap(err, "error calculating relative bramblefile loc") + } + runLocation = filepath.Join(destination, relBramblefileLocation) + if err = os.MkdirAll(runLocation, 0755); err != nil { + return "", errors.Wrap(err, "error making build directory") + } + drv.Env["src"] = destination + return +} + +func (drv *Derivation) writeDerivation() (err error) { + fileBytes, filename, err := drv.computeDerivation() + if err != nil { + return + } + fileLocation := drv.function.joinStorePath(filename) + + if !pathExists(fileLocation) { + return ioutil.WriteFile(fileLocation, fileBytes, 0444) + } + return nil +} + +func (drv *Derivation) storePath() string { return drv.function.bramble.StorePath() } + +func (drv *Derivation) createBuildDir() (tempDir string, err error) { + return ioutil.TempDir("", TempDirPrefix) +} + +func (drv *Derivation) computeOutPath() (outPath string, err error) { + _, filename, err := drv.computeDerivation() + + return filepath.Join( + drv.storePath(), + strings.TrimSuffix(filename, ".drv"), + ), err +} + +func (drv *Derivation) expand(s string) string { + return os.Expand(s, func(i string) string { + if i == "bramble_path" { + return drv.storePath() + } + if v, ok := drv.Env[i]; ok { + return v + } + return "" + }) +} + +func (drv *Derivation) regularBuilder(buildDir, outPath string) (err error) { + var runLocation string + runLocation, err = drv.assembleSources(buildDir) + if err != nil { + return + } + builderLocation := drv.expand(drv.Builder) + + if _, err := os.Stat(builderLocation); err != nil { + return errors.Wrap(err, "error checking if builder location exists") + } + cmd := exec.Command(builderLocation, drv.Args...) + cmd.Dir = runLocation + cmd.Env = []string{} + for k, v := range drv.Env { + v = strings.Replace(v, "$bramble_path", drv.storePath(), -1) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "out", outPath)) + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "bramble_path", drv.storePath())) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func (drv *Derivation) fetchURLBuilder(outPath string) (err error) { + url, ok := drv.Env["url"] + if !ok { + return errors.New("fetch_url requires the environment variable 'url' to be set") + } + hash, ok := drv.Env["hash"] + if !ok { + return errors.New("fetch_url requires the environment variable 'hash' to be set") + } + path, err := drv.function.DownloadFile(url, hash) + if err != nil { + return err + } + // TODO: what if this package changes? + if err = archiver.Unarchive(path, outPath); err != nil { + return errors.Wrap(err, "error unarchiving") + } + return nil +} + +func (drv *Derivation) functionBuilder(buildDir, outPath string) (err error) { + var meta functionBuilderMeta + + if err = json.Unmarshal([]byte(drv.Env["function_builder_meta"]), &meta); err != nil { + return errors.Wrap(err, "error parsing function_builder_meta") + } + session, err := newSession(buildDir, drv.Env) + if err != nil { + return + } + session.setEnv("out", outPath) + for k, v := range session.env { + session.env[k] = strings.Replace(v, "$bramble_path", drv.storePath(), -1) + } + spew.Dump(session) + return drv.function.bramble.CallInlineDerivationFunction( + meta, session, + ) +} + +func (drv *Derivation) build() (err error) { + buildDir, err := drv.createBuildDir() + if err != nil { + return + } + + outPath, err := drv.computeOutPath() + if err != nil { + return err + } + if err = os.MkdirAll(outPath, 0755); err != nil { + return + } + + switch drv.Builder { + case "fetch_url": + err = drv.fetchURLBuilder(outPath) + case "function": + err = drv.functionBuilder(buildDir, outPath) + default: + err = drv.regularBuilder(buildDir, outPath) + } + if err != nil { + return + } + + matches, hashString, err := drv.hashAndScanDirectory(outPath) + if err != nil { + return + } + folderName := hashString + "-" + drv.Name + drv.Outputs["out"] = Output{Path: folderName, Dependencies: matches} + + newPath := drv.function.joinStorePath() + "/" + folderName + _, doesnotExistErr := os.Stat(newPath) + drv.function.log.Debug("Output at ", newPath) + if doesnotExistErr != nil { + return os.Rename(outPath, newPath) + } + // hashed content is already there, just exit + return +} + +func (drv *Derivation) hashAndScanDirectory(location string) (matches []string, hashString string, err error) { + var storeValues []string + old := drv.storePath() + new := BramblePrefixOfRecord + + for _, derivation := range drv.function.derivations { + storeValues = append(storeValues, strings.Replace(derivation.String(), "$bramble_path", old, 1)) + } + + errChan := make(chan error) + resultChan := make(chan map[string]struct{}) + pipeReader, pipeWriter := io.Pipe() + + go func() { + if err := reptar.Reptar(location, pipeWriter); err != nil { + errChan <- err + } + if err = pipeWriter.Close(); err != nil { + errChan <- err + } + }() + hasher := NewHasher() + go func() { + _, matches, err := textreplace.ReplaceStringsPrefix(pipeReader, hasher, storeValues, old, new) + if err != nil { + errChan <- err + } + resultChan <- matches + }() + select { + case err := <-errChan: + return nil, "", err + case result := <-resultChan: + for k := range result { + matches = append(matches, strings.Replace(k, drv.storePath(), "$bramble_path", 1)) + } + return matches, hasher.String(), nil + } +} + +func commonFilepathPrefix(paths []string) string { + sep := byte(os.PathSeparator) + if len(paths) == 0 { + return string(sep) + } + + c := []byte(path.Clean(paths[0])) + c = append(c, sep) + + for _, v := range paths[1:] { + v = path.Clean(v) + string(sep) + if len(v) < len(c) { + c = c[:len(v)] + } + for i := 0; i < len(c); i++ { + if v[i] != c[i] { + c = c[:i] + break + } + } + } + + for i := len(c) - 1; i >= 0; i-- { + if c[i] == sep { + c = c[:i+1] + break + } + } + + return string(c) +} + +// Hasher is used to compute path hash values. Hasher implements io.Writer and +// takes a sha256 hash of the input bytes. The output string is a lowercase +// base32 representation of the first 160 bits of the hash +type Hasher struct { + hash hash.Hash +} + +func NewHasher() *Hasher { + return &Hasher{ + hash: sha256.New(), + } +} + +func (h *Hasher) Write(b []byte) (n int, err error) { + return h.hash.Write(b) +} + +func (h *Hasher) String() string { + return bytesToBase32Hash(h.hash.Sum(nil)) +} + +// bytesToBase32Hash copies nix here +// https://nixos.org/nixos/nix-pills/nix-store-paths.html +// Finally the comments tell us to compute the base32 representation of the +// first 160 bits (truncation) of a sha256 of the above string: +func bytesToBase32Hash(b []byte) string { + var buf bytes.Buffer + _, _ = base32.NewEncoder(base32.StdEncoding, &buf).Write(b[:20]) + return strings.ToLower(buf.String()) +} + +func hashFile(name string, file io.ReadCloser) (fileHash, filename string, err error) { + defer file.Close() + hasher := NewHasher() + if _, err = hasher.Write([]byte(name)); err != nil { + return + } + if _, err = io.Copy(hasher, file); err != nil { + return + } + filename = fmt.Sprintf("%s-%s", hasher.String(), name) + return +} + +// CopyFiles takes a list of absolute paths to files and copies them into +// another directory, maintaining structure +func copyFiles(prefix string, files []string, dest string) (err error) { + files, err = expandPathDirectories(files) + if err != nil { + return err + } + + sort.Slice(files, func(i, j int) bool { return len(files[i]) < len(files[j]) }) + for _, file := range files { + destPath := filepath.Join(dest, strings.TrimPrefix(file, prefix)) + fileInfo, err := os.Stat(file) + if err != nil { + return errors.Wrap(err, "error finding source file") + } + + stat, ok := fileInfo.Sys().(*syscall.Stat_t) + if !ok { + return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", file) + } + + switch fileInfo.Mode() & os.ModeType { + case os.ModeDir: + if err := createDirIfNotExists(destPath, 0755); err != nil { + return err + } + case os.ModeSymlink: + if err := copySymLink(file, destPath); err != nil { + return err + } + default: + if err := copyFile(file, destPath); err != nil { + return err + } + } + + if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + // TODO: when does this happen??? + isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 + if !isSymlink { + if err := os.Chmod(destPath, fileInfo.Mode()); err != nil { + return err + } + } + } + return +} + +// takes a list of paths and adds all files in all subdirectories +func expandPathDirectories(files []string) (out []string, err error) { + for _, file := range files { + if err = filepath.Walk(file, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + out = append(out, path) + return nil + }); err != nil { + return + } + } + return +} + +func copyFile(srcFile, dstFile string) error { + out, err := os.Create(dstFile) + if err != nil { + return err + } + + defer out.Close() + + in, err := os.Open(srcFile) + if err != nil { + return err + } + defer in.Close() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return nil +} + +func createDirIfNotExists(dir string, perm os.FileMode) error { + if pathExists(dir) { + return nil + } + + if err := os.MkdirAll(dir, perm); err != nil { + return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) + } + + return nil +} + +func copySymLink(source, dest string) error { + link, err := os.Readlink(source) + if err != nil { + return err + } + return os.Symlink(link, dest) +} diff --git a/pkg/bramble/integration_test.go b/pkg/bramble/integration_test.go index 08da2515..b858b299 100644 --- a/pkg/bramble/integration_test.go +++ b/pkg/bramble/integration_test.go @@ -1,11 +1,43 @@ package bramble -import "testing" +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/alecthomas/assert" +) + +var ( + TestTmpDirPrefix = "bramble-test-" +) func TestIntegration(t *testing.T) { b := Bramble{} + dir, err := ioutil.TempDir("", TestTmpDirPrefix) + assert.NoError(t, err) + // set a unique bramble store for these tests + os.Setenv("BRAMBLE_PATH", dir) if err := b.test([]string{"../../tests"}); err != nil { + fmt.Printf("%+v", err) + t.Error(err) + } + + _ = os.RemoveAll(dir) +} + +func TestRunStarlarkBuilder(t *testing.T) { + b := Bramble{} + + dir, err := ioutil.TempDir("", TestTmpDirPrefix) + assert.NoError(t, err) + // set a unique bramble store for these tests + os.Setenv("BRAMBLE_PATH", dir) + if err := b.run([]string{"../../tests/starlark-builder:run_busybox"}); err != nil { t.Error(err) } + + _ = os.RemoveAll(dir) } diff --git a/pkg/bramble/os.go b/pkg/bramble/os.go new file mode 100644 index 00000000..d8317425 --- /dev/null +++ b/pkg/bramble/os.go @@ -0,0 +1,116 @@ +package bramble + +import ( + "bufio" + goos "os" + + "github.com/maxmcd/bramble/pkg/assert" + "github.com/maxmcd/bramble/pkg/starutil" + "go.starlark.net/starlark" +) + +type OS struct { + bramble *Bramble + session *session +} + +var ( + _ starlark.Value = OS{} + _ starlark.HasAttrs = OS{} +) + +func NewOS(bramble *Bramble, session *session) OS { + return OS{bramble: bramble, session: session} +} + +func (os OS) String() string { return "" } +func (os OS) Freeze() {} +func (os OS) Type() string { return "module" } +func (os OS) Truth() starlark.Bool { return starlark.True } +func (os OS) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("os") } +func (os OS) AttrNames() []string { + return []string{ + "args", + "error", + "input", + "mkdir", + } +} + +func makeArgs() (starlark.Value, error) { + out := []starlark.Value{} + if len(goos.Args) >= 3 { + for _, arg := range goos.Args[3:] { + out = append(out, starlark.String(arg)) + } + } + return starlark.NewList(out), nil +} + +func (os OS) Attr(name string) (val starlark.Value, err error) { + os.bramble.AfterDerivation() + switch name { + case "args": + return makeArgs() + case "error": + return starutil.Callable{ThisName: "error", ParentName: "os", Callable: os.error}, nil + case "cd": + return starutil.Callable{ThisName: "cd", ParentName: "os", Callable: os.cd}, nil + case "getenv": + return starutil.Callable{ThisName: "getenv", ParentName: "os", Callable: os.getenv}, nil + case "setenv": + return starutil.Callable{ThisName: "setenv", ParentName: "os", Callable: os.setenv}, nil + case "input": + return starutil.Callable{ThisName: "input", ParentName: "os", Callable: os.input}, nil + case "mkdir": + return starutil.Callable{ThisName: "mkdir", ParentName: "os", Callable: os.mkdir}, nil + } + return nil, nil +} + +func (os OS) error(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + return assert.Error(thread, nil, args, kwargs) +} + +func (os OS) input(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + reader := bufio.NewReader(goos.Stdin) + text, err := reader.ReadString('\n') + return starlark.String(text), err +} + +func (os OS) mkdir(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + var path starlark.String + if err = starlark.UnpackArgs("mkdir", args, kwargs, "path", &path); err != nil { + return + } + return starlark.None, goos.Mkdir(os.session.expand(path.GoString()), 0755) +} + +func (os OS) getenv(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + var key starlark.String + if err = starlark.UnpackArgs("getenv", args, kwargs, "key", &key); err != nil { + return + } + return starlark.String(os.session.getEnv(key.GoString())), nil +} + +func (os OS) setenv(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + var key starlark.String + var value starlark.String + if err = starlark.UnpackArgs("setenv", args, kwargs, "key", &key, "value", &value); err != nil { + return + } + os.session.setEnv( + key.GoString(), + value.GoString(), + ) + return starlark.None, nil +} + +func (os OS) cd(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { + var path starlark.String + if err = starlark.UnpackArgs("cd", args, kwargs, "path", &path); err != nil { + return + } + return starlark.None, os.session.cd(os.session.expand(path.GoString())) +} diff --git a/pkg/bramble/session.go b/pkg/bramble/session.go new file mode 100644 index 00000000..f3a3b7cf --- /dev/null +++ b/pkg/bramble/session.go @@ -0,0 +1,66 @@ +package bramble + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +type session struct { + env map[string]string + currentDirectory string +} + +func newSession(wd string, env map[string]string) (*session, error) { + env2 := map[string]string{} + if env == nil { + for _, kv := range os.Environ() { + parts := strings.SplitN(kv, "=", 1) + if len(parts) > 1 { + env2[parts[0]] = parts[1] + } else { + env2[parts[0]] = "" + } + } + } else { + // explicitly copy so that we're not editing another structure somewhere + for k, v := range env { + env2[k] = v + } + } + s := session{env: env2} + var err error + // if wd="" filepath.Abs will get the absolute path to the current working + // directory + s.currentDirectory, err = filepath.Abs(wd) + return &s, err +} + +func (s *session) expand(in string) string { + return os.Expand(in, s.getEnv) +} + +func (s *session) getEnv(key string) string { + return s.env[s.expand(key)] +} +func (s *session) setEnv(key, value string) { + s.env[s.expand(key)] = s.expand(value) +} + +func (s *session) envArray() (out []string) { + for k, v := range s.env { + out = append(out, fmt.Sprintf("%s=%s", k, v)) + } + return +} + +func (s *session) cd(path string) (err error) { + if strings.HasPrefix(path, "/") { + s.currentDirectory = path + } else { + s.currentDirectory = filepath.Join(s.currentDirectory, path) + } + _, err = os.Stat(s.currentDirectory) + return +} diff --git a/pkg/derivation/path.go b/pkg/bramble/store.go similarity index 92% rename from pkg/derivation/path.go rename to pkg/bramble/store.go index d3bc8921..e64882e9 100644 --- a/pkg/derivation/path.go +++ b/pkg/bramble/store.go @@ -1,4 +1,4 @@ -package derivation +package bramble import ( "fmt" @@ -9,6 +9,13 @@ import ( "github.com/pkg/errors" ) +var ( + PathPaddingCharacters = "bramble_store_padding" + PathPaddingLength = 50 + + ErrStoreDoesNotExist = errors.New("calculated store path doesn't exist, did the location change?") +) + func ensureBramblePath() (bramblePath, storePath string, err error) { bramblePath = os.Getenv("BRAMBLE_PATH") if bramblePath == "" { diff --git a/pkg/derivation/path_test.go b/pkg/bramble/store_test.go similarity index 98% rename from pkg/derivation/path_test.go rename to pkg/bramble/store_test.go index dbd5b733..831cf20e 100644 --- a/pkg/derivation/path_test.go +++ b/pkg/bramble/store_test.go @@ -1,4 +1,4 @@ -package derivation +package bramble import ( "path/filepath" diff --git a/pkg/bramblecmd/README.md b/pkg/bramblecmd/README.md deleted file mode 100644 index 0c961c43..00000000 --- a/pkg/bramblecmd/README.md +++ /dev/null @@ -1,7 +0,0 @@ - - - -- command starts to run -- stdout and stderr are written to a buffer with a limit, stout/err writes will block when the buffer is full -- user can pass print_output option if they want to see the output -- diff --git a/pkg/bramblecmd/bramblescript.go b/pkg/bramblecmd/bramblescript.go deleted file mode 100644 index c46a9b6a..00000000 --- a/pkg/bramblecmd/bramblescript.go +++ /dev/null @@ -1,24 +0,0 @@ -package bramblecmd - -import ( - "errors" - "io/ioutil" - "log" - - "go.starlark.net/starlark" -) - -var logger *log.Logger -var ( - ErrInvalidRead = errors.New("can't read from command output more than once") -) - -func init() { - logger = log.New(ioutil.Discard, "", 0) -} - -func Builtins(dir string) starlark.StringDict { - return starlark.StringDict{ - "cmd": NewFunction(), - } -} diff --git a/pkg/bramblecmd/bramblescript_test.go b/pkg/bramblecmd/bramblescript_test.go deleted file mode 100644 index af948424..00000000 --- a/pkg/bramblecmd/bramblescript_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package bramblecmd - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "go.starlark.net/starlark" -) - -type scriptTest struct { - name string - script string - errContains string - returnValue string -} - -func runTest(t *testing.T, tests []scriptTest) { - for _, tt := range tests { - t.Run(tt.script, func(t *testing.T) { - thread := &starlark.Thread{Name: "main"} - globals, err := starlark.ExecFile(thread, tt.name+".bramble", tt.script, Builtins("")) - if err != nil || tt.errContains != "" { - if err == nil { - t.Error("error is nil") - return - } - assert.Contains(t, err.Error(), tt.errContains) - if tt.errContains == "" { - t.Error(err, tt.script) - return - } - } - if tt.returnValue == "" { - return - } - b, ok := globals["b"] - if !ok { - t.Errorf("%q doesn't output global value b", tt.script) - return - } - assert.Equal(t, tt.returnValue, b.String()) - }) - } -} diff --git a/pkg/bramblecmd/buffered_pipe.go b/pkg/bramblecmd/buffered_pipe.go deleted file mode 100644 index ba05e578..00000000 --- a/pkg/bramblecmd/buffered_pipe.go +++ /dev/null @@ -1,73 +0,0 @@ -package bramblecmd - -import ( - "io" - "sync" - "syscall" -) - -// https://github.com/golang/go/blob/0436b162397018c45068b47ca1b5924a3eafdee0/src/net/net_fake.go#L173 - -func newBufferedPipe(softLimit int) *bufferedPipe { - p := &bufferedPipe{softLimit: softLimit} - p.rCond.L = &p.mu - p.wCond.L = &p.mu - return p -} - -type bufferedPipe struct { - softLimit int - mu sync.Mutex - buf []byte - closed bool - rCond sync.Cond - wCond sync.Cond -} - -func (p *bufferedPipe) Read(b []byte) (int, error) { - p.mu.Lock() - defer p.mu.Unlock() - for { - if p.closed && len(p.buf) == 0 { - return 0, io.EOF - } - if len(p.buf) > 0 { - break - } - p.rCond.Wait() - } - - n := copy(b, p.buf) - p.buf = p.buf[n:] - p.wCond.Broadcast() - return n, nil -} - -func (p *bufferedPipe) Write(b []byte) (int, error) { - p.mu.Lock() - defer p.mu.Unlock() - - for { - if p.closed { - return 0, syscall.ENOTCONN - } - if len(p.buf) <= p.softLimit { - break - } - p.wCond.Wait() - } - - p.buf = append(p.buf, b...) - p.rCond.Broadcast() - return len(b), nil -} - -func (p *bufferedPipe) Close() (err error) { - p.mu.Lock() - defer p.mu.Unlock() - - p.closed = true - p.rCond.Broadcast() - p.wCond.Broadcast() - return nil -} diff --git a/pkg/bramblecmd/bytestream.go b/pkg/bramblecmd/bytestream.go deleted file mode 100644 index 783635bb..00000000 --- a/pkg/bramblecmd/bytestream.go +++ /dev/null @@ -1,90 +0,0 @@ -package bramblecmd - -import ( - "bufio" - "errors" - "fmt" - "io" - "io/ioutil" - - "go.starlark.net/starlark" -) - -type ByteStream struct { - cmd *Cmd - stdout bool - stderr bool -} - -var ( - _ starlark.Value = ByteStream{} - _ starlark.Callable = ByteStream{} - _ starlark.Iterable = ByteStream{} -) - -func (bs ByteStream) Name() string { - if bs.stdout && bs.stderr { - return "output" - } - if bs.stderr { - return "stderr" - } - return "stdout" -} -func (bs ByteStream) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { - if err = bs.cmd.setOutput(bs.stdout, bs.stderr); err != nil { - return - } - b, err := ioutil.ReadAll(bs.cmd) - if err == io.ErrClosedPipe { - err = nil - } - if err != nil { - // TODO: need a better solution for this - fmt.Println(string(b)) - } - return starlark.String(string(b)), err -} - -func (bs ByteStream) String() string { - return fmt.Sprintf("", bs.Name()) -} - -func (bs ByteStream) Type() string { return "bytestream" } -func (bs ByteStream) Freeze() {} -func (bs ByteStream) Truth() starlark.Bool { return bs.cmd.Truth() } -func (bs ByteStream) Hash() (uint32, error) { return 0, errors.New("bytestream is unhashable") } - -func (bs ByteStream) Iterate() starlark.Iterator { - err := bs.cmd.setOutput(bs.stdout, bs.stderr) - bsi := byteStreamIterator{ - bs: bs, - } - if err == nil { - bsi.buf = bufio.NewReader(bs.cmd) - } - return bsi -} - -type byteStreamIterator struct { - bs ByteStream - buf *bufio.Reader -} - -func (bsi byteStreamIterator) Next(p *starlark.Value) bool { - if bsi.buf == nil { - return false - } - str, err := bsi.buf.ReadString('\n') - if err == io.EOF || err == io.ErrClosedPipe { - return false - } - if err != nil { - // TODO: something better here? certain errors we care about? - panic(err) - } - *p = starlark.String(str[:len(str)-1]) - return true -} - -func (bsi byteStreamIterator) Done() {} diff --git a/pkg/bramblecmd/bytestream_test.go b/pkg/bramblecmd/bytestream_test.go deleted file mode 100644 index 2ce37a9e..00000000 --- a/pkg/bramblecmd/bytestream_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package bramblecmd - -import "testing" - -func TestByteStream(t *testing.T) { - tests := []scriptTest{ - {script: `b=cmd("ls", "notathing").stdout()`, - errContains: `exit`}, - {script: `b=cmd("echo","hi").stdout()`, - returnValue: `"hi\n"`}, - {script: `b=list(cmd("echo","hi").stdout)`, - returnValue: `["hi"]`}, - {script: `b=cmd("echo","hi").stdout`, - returnValue: ``}, - {script: `b=type(cmd("echo","hi").stdout)`, - returnValue: `"bytestream"`}, - {script: `b=cmd("echo", "hi").output()`, - returnValue: `"hi\n"`}, - {script: `b=cmd("echo", "hi").stderr()`, - returnValue: `""`}, - {script: `b=cmd("echo", "hi").stderr`, - returnValue: ``}, - {script: `b=cmd("echo", "hi").stdout`, - returnValue: ``}, - {script: `b=cmd("echo", "hi").output`, - returnValue: ``}, - {script: `b=cmd("echo", 1).stdout()`, - returnValue: `"1\n"`}, - {script: ` -def run(): - c = cmd("ls") - response = "" - for line in c.stdout: - if "cmd_test" in line: - response = line - return response -b = run()`, - returnValue: `"cmd_test.go"`}, - {script: ` -def run(): - c = cmd("ls") - for line in c.output: - if "cmd_test" in line: - return line -b = run()`, - returnValue: `"cmd_test.go"`}, - } - runTest(t, tests) -} diff --git a/pkg/bramblecmd/cmd_test.go b/pkg/bramblecmd/cmd_test.go deleted file mode 100644 index 2d407911..00000000 --- a/pkg/bramblecmd/cmd_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package bramblecmd - -import ( - "testing" -) - -func TestStarlarkCmd(t *testing.T) { - tests := []scriptTest{ - {script: ` -c = cmd("ls") -b = [getattr(c, x) for x in dir(c)] - `, - returnValue: ""}, - {script: "cmd()", - errContains: "missing 1 required positional argument"}, - {script: "cmd([])", - errContains: "be empty"}, - {script: `cmd("")`, - errContains: "be empty"}, - {script: `cmd(" ")`, - errContains: `" "`}, - {script: `b=[getattr(cmd("echo"), x) for x in dir(cmd("echo"))]`, - returnValue: ``}, - {script: `cmd("sleep 2").kill()`}, - {script: `b=cmd(["ls", "-lah"])`, - returnValue: ``}, - {script: `b=cmd("ls -lah")`, - returnValue: ``}, - {script: `b=cmd("ls \"-lah\"")`, - returnValue: ``}, - {script: `b=cmd("ls '-lah'")`, - returnValue: ``}, - {script: `b=cmd("ls", "-lah")`, - returnValue: ``}, - {script: `b=cmd("echo 'these are words'").pipe("tr ' ' '\n'").pipe("grep these").stdout()`, - returnValue: `"these\n"`}, - {script: `b=cmd("ls -lah").wait().exit_code`, - returnValue: `0`}, - {script: `c = cmd("echo") -cmd("echo").wait() -c.kill()`}, - } - runTest(t, tests) -} - -func TestPipe(t *testing.T) { - tests := []scriptTest{ - {script: `b=cmd("echo 'these are words'").pipe("tr ' ' '\n'").pipe("grep these").stdout()`, - returnValue: `"these\n"`}, - } - runTest(t, tests) -} - -func TestCallback(t *testing.T) { - tests := []scriptTest{ - {script: ` -def echo(*args, **kwargs): - return cmd("echo", *args, **kwargs) - -b=echo("hi").stdout().strip() -`, - returnValue: `"hi"`}, - } - runTest(t, tests) -} - -func TestArgs(t *testing.T) { - tests := []scriptTest{ - {script: `b=cmd("grep hi", stdin=cmd("echo hi")).output()`, - returnValue: `"hi\n"`}, - {script: `b=cmd("grep hi", stdin="hi").output()`, - returnValue: `"hi\n"`}, - {script: `b=cmd("env", clear_env=True).output()`, - returnValue: `""`}, - {script: `b=cmd("env", clear_env=True, env={"foo":"bar", "baz": 1}).output()`, - returnValue: `"foo=bar\nbaz=1\n"`}, - } - runTest(t, tests) -} - -func TestIfErr(t *testing.T) { - tests := []scriptTest{ - {script: `b=cmd("ls", "notathing").if_err("echo", "hi").stdout()`, - returnValue: `"hi\n"`}, - {script: `b=cmd("ls", "notathing").if_err("echo", "hi").stdout()`, - returnValue: `"hi\n"`}, - } - runTest(t, tests) -} - -func TestCallable(t *testing.T) { - tests := []scriptTest{ - {script: `b=cmd("ls").pipe`, - returnValue: ``}, - {script: `b=type(cmd("ls").if_err)`, - returnValue: `"builtin_function_or_method"`}, - } - runTest(t, tests) -} diff --git a/pkg/bramblecmd/examples/imports_scratch.bramble b/pkg/bramblecmd/examples/imports_scratch.bramble deleted file mode 100644 index e372bcc7..00000000 --- a/pkg/bramblecmd/examples/imports_scratch.bramble +++ /dev/null @@ -1,12 +0,0 @@ -""" -fake code that tries to reason through how imports work -""" - -load("nomad") - -# loads a struct with all global methods - - -def main(args): - cmd.cd("..") - nomad.deploy() diff --git a/pkg/bramblecmd/examples/new_structure/main.bramble b/pkg/bramblecmd/examples/new_structure/main.bramble deleted file mode 100644 index 06a2ef2e..00000000 --- a/pkg/bramblecmd/examples/new_structure/main.bramble +++ /dev/null @@ -1,40 +0,0 @@ -"""tour of some features and limitations""" - - -def ls(): - assert.eq(1, 1) - assert.ne(1, 2) - print(os.args) - return run_and_print("ls", *os.args) - - -def get_something(): - print("What is your name?") - thing = os.input() - - # clear previous line - print("\033[A \033[A") - print("hello %s" % thing) - - - -def build_flask(): - cmd("python build").wait() - -def flask(): - foo = derivation( - script=build_flask, - src="./" - ) - - cmd(foo, "python", "flask") - - - -def run_and_print(*args, **kwargs): - """run and print""" - print(*args) - runner = cmd(*args, **kwargs) - out = runner.output() - print(out) - return runner diff --git a/pkg/bramblecmd/examples/new_structure/main_test.bramble b/pkg/bramblecmd/examples/new_structure/main_test.bramble deleted file mode 100644 index 478b7c04..00000000 --- a/pkg/bramblecmd/examples/new_structure/main_test.bramble +++ /dev/null @@ -1,5 +0,0 @@ -load("github.com/maxmcd/bramble/pkg/bramblecmd/examples/new_structure/main", "ls") - -def test_ls(): - ran = ls() - assert.eq(ran.exit_code, 0) diff --git a/pkg/bramblecmd/examples/nomad_deploy.bramble b/pkg/bramblecmd/examples/nomad_deploy.bramble deleted file mode 100644 index f70f7e27..00000000 --- a/pkg/bramblecmd/examples/nomad_deploy.bramble +++ /dev/null @@ -1,34 +0,0 @@ -"""deploy with nomad""" - - -def main(args): - # Where do these args come from? - - if len(args) == 0: - # raise an error? - pass - file = args[0] - - contents = cmd("cat %s" % file).stdout() - # optionally replace the contents? - - plan = cmd("nomad job plan -", stdin=contents).wait() - - out = plan.output() - - print(out) - # ask for confirmation? - - index = None - - for line in out.split("\n"): - if "Job Modify Index" in line: - index = line.split(" ")[-1] - - if index == None: - # raise error? - pass - - run = cmd("nomad job run -check-index %s %s" % (index, file)).wait() - - # continue from here and diff --git a/pkg/bramblecmd/examples/other_new_structure/main.bramble b/pkg/bramblecmd/examples/other_new_structure/main.bramble deleted file mode 100644 index 4f48c297..00000000 --- a/pkg/bramblecmd/examples/other_new_structure/main.bramble +++ /dev/null @@ -1,5 +0,0 @@ -load("github.com/maxmcd/bramble/pkg/bramblecmd/examples/new_structure/main", foo="ls") - - -def main(): - foo() diff --git a/pkg/bramblecmd/examples/run.bramble b/pkg/bramblecmd/examples/run.bramble deleted file mode 100755 index 8dc1a6f5..00000000 --- a/pkg/bramblecmd/examples/run.bramble +++ /dev/null @@ -1,13 +0,0 @@ -def remove_empty(arr): - return [a for a in arr if a != ""] - - -def total_bytes_in_folder(): - total = 0 - for line in cmd("ls -Rla .").stdout: - rows = remove_empty(line.split(" ")) - if len(rows) < 3: - continue - if "d" not in rows[0]: - total += int(rows[4]) - print(total, "bytes") diff --git a/pkg/bramblecmd/examples/vault.bramble b/pkg/bramblecmd/examples/vault.bramble deleted file mode 100644 index cbef8b7c..00000000 --- a/pkg/bramblecmd/examples/vault.bramble +++ /dev/null @@ -1,26 +0,0 @@ -""" -script to delete all vault tokens with manager group permissions -""" - - -def vault_cmd(token=None): - def _vault(*args, **kwargs): - return cmd("vault", *args, env={"VAULT_TOKEN": token}, **kwargs) - - return _vault - - -def main(): - # how to set this token? - token = "?????" - vault = vault_cmd(token) - values = list(vault("list", "auth/token/accessors").output) - manager_tokens = [] - for value in values: - if ( - "default manager" - in vault("token", "lookup", "-accessor", value).output() - ): - manager_tokens.append(value) - for token in manager_tokens: - vault("token", "revoke", "-accessor", token).wait() diff --git a/pkg/bramblecmd/function.go b/pkg/bramblecmd/function.go deleted file mode 100644 index 2b93cfe0..00000000 --- a/pkg/bramblecmd/function.go +++ /dev/null @@ -1,31 +0,0 @@ -package bramblecmd - -import ( - "github.com/maxmcd/bramble/pkg/starutil" - "go.starlark.net/starlark" -) - -// Function is the value for the builtin "cmd", calling it as a function -// creates a new cmd instance, it also has various other attributes and methods -type Function struct{} - -var ( - _ starlark.Value = new(Function) - _ starlark.Callable = new(Function) -) - -func NewFunction() *Function { - return &Function{} -} - -func (fn *Function) Freeze() {} -func (fn *Function) Hash() (uint32, error) { return 0, starutil.ErrUnhashable(fn.Type()) } -func (fn *Function) Name() string { return fn.Type() } -func (fn *Function) String() string { return "" } -func (fn *Function) Type() string { return "builtin_function_cmd" } -func (fn *Function) Truth() starlark.Bool { return true } - -// CallInternal defines the cmd() starlark function. -func (fn *Function) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (v starlark.Value, err error) { - return newCmd(thread, args, kwargs, nil, "") -} diff --git a/pkg/bramblecmd/refactor.go b/pkg/bramblecmd/refactor.go deleted file mode 100644 index 327e531f..00000000 --- a/pkg/bramblecmd/refactor.go +++ /dev/null @@ -1 +0,0 @@ -package bramblecmd diff --git a/pkg/bramblecmd/refactor_test.go b/pkg/bramblecmd/refactor_test.go deleted file mode 100644 index 327e531f..00000000 --- a/pkg/bramblecmd/refactor_test.go +++ /dev/null @@ -1 +0,0 @@ -package bramblecmd diff --git a/pkg/bramblecmd/standard_stream.go b/pkg/bramblecmd/standard_stream.go deleted file mode 100644 index 01c9dbc9..00000000 --- a/pkg/bramblecmd/standard_stream.go +++ /dev/null @@ -1,73 +0,0 @@ -package bramblecmd - -import ( - "io" - "io/ioutil" - "sync" - - "github.com/moby/moby/pkg/stdcopy" -) - -type StandardStream struct { - stdout bool - stderr bool - once sync.Once - - buffPipe *bufferedPipe - - reader *io.PipeReader - - err error - lock sync.Mutex - - started bool -} - -func NewStandardStream() (ss *StandardStream, stdoutWriter, stderrWriter io.Writer) { - ss = &StandardStream{ - buffPipe: newBufferedPipe(4096 * 16), - } - - stdoutWriter = stdcopy.NewStdWriter(ss.buffPipe, stdcopy.Stdout) - stderrWriter = stdcopy.NewStdWriter(ss.buffPipe, stdcopy.Stderr) - - return -} - -func (ss *StandardStream) Close() (err error) { - _ = ss.buffPipe.Close() - return nil -} - -func (ss *StandardStream) Read(p []byte) (n int, err error) { - ss.once.Do(func() { - ss.started = true - var writer *io.PipeWriter - ss.lock.Lock() - ss.reader, writer = io.Pipe() - ss.lock.Unlock() - var stdoutWriter io.Writer = ioutil.Discard - var stderrWriter io.Writer = ioutil.Discard - if ss.stdout { - stdoutWriter = writer - } - if ss.stderr { - stderrWriter = writer - } - go func() { - _, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, ss.buffPipe) - ss.lock.Lock() - ss.err = err - ss.lock.Unlock() - _ = writer.Close() - _ = ss.reader.Close() - }() - }) - n, err = ss.reader.Read(p) - ss.lock.Lock() - if err == nil && ss.err != nil { - err = ss.err - } - ss.lock.Unlock() - return -} diff --git a/pkg/brambleos/os.go b/pkg/brambleos/os.go deleted file mode 100644 index 03304a9d..00000000 --- a/pkg/brambleos/os.go +++ /dev/null @@ -1,64 +0,0 @@ -package brambleos - -import ( - "bufio" - goos "os" - - "github.com/maxmcd/bramble/pkg/starutil" - "go.starlark.net/starlark" -) - -type OS struct{} - -var ( - _ starlark.Value = OS{} - _ starlark.HasAttrs = OS{} -) - -func (os OS) String() string { return "" } -func (os OS) Freeze() {} -func (os OS) Type() string { return "module" } -func (os OS) Truth() starlark.Bool { return starlark.True } -func (os OS) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("os") } -func (os OS) AttrNames() []string { - return []string{ - "args", - "error", - "input", - } -} - -func makeArgs() (starlark.Value, error) { - out := []starlark.Value{} - if len(goos.Args) >= 3 { - for _, arg := range goos.Args[3:] { - out = append(out, starlark.String(arg)) - } - } - return starlark.NewList(out), nil -} - -func (os OS) Attr(name string) (val starlark.Value, err error) { - switch name { - case "args": - return makeArgs() - case "error": - return starutil.Callable{ThisName: "error", ParentName: "os", Callable: os.error}, nil - case "input": - return starutil.Callable{ThisName: "input", ParentName: "os", Callable: os.input}, nil - } - return nil, nil -} - -func (os OS) error(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { - if err = starlark.UnpackArgs("args", args, kwargs, "val", &val); err != nil { - return - } - panic(val) //TODO -} - -func (os OS) input(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (val starlark.Value, err error) { - reader := bufio.NewReader(goos.Stdin) - text, err := reader.ReadString('\n') - return starlark.String(text), err -} diff --git a/pkg/derivation/common_prefix.go b/pkg/derivation/common_prefix.go deleted file mode 100644 index 471c9761..00000000 --- a/pkg/derivation/common_prefix.go +++ /dev/null @@ -1,38 +0,0 @@ -package derivation - -import ( - "os" - "path" -) - -func commonPrefix(paths []string) string { - sep := byte(os.PathSeparator) - if len(paths) == 0 { - return string(sep) - } - - c := []byte(path.Clean(paths[0])) - c = append(c, sep) - - for _, v := range paths[1:] { - v = path.Clean(v) + string(sep) - if len(v) < len(c) { - c = c[:len(v)] - } - for i := 0; i < len(c); i++ { - if v[i] != c[i] { - c = c[:i] - break - } - } - } - - for i := len(c) - 1; i >= 0; i-- { - if c[i] == sep { - c = c[:i+1] - break - } - } - - return string(c) -} diff --git a/pkg/derivation/copy_files.go b/pkg/derivation/copy_files.go deleted file mode 100644 index e90b043d..00000000 --- a/pkg/derivation/copy_files.go +++ /dev/null @@ -1,131 +0,0 @@ -package derivation - -import ( - "fmt" - "io" - "os" - "path/filepath" - "sort" - "strings" - "syscall" - - "github.com/pkg/errors" -) - -// CopyFiles takes a list of absolute paths to files and copies them into -// another directory, maintaining structure -func copyFiles(prefix string, files []string, dest string) (err error) { - files, err = expandPathDirectories(files) - if err != nil { - return err - } - - sort.Slice(files, func(i, j int) bool { return len(files[i]) < len(files[j]) }) - for _, file := range files { - destPath := filepath.Join(dest, strings.TrimPrefix(file, prefix)) - fileInfo, err := os.Stat(file) - if err != nil { - return errors.Wrap(err, "error finding source file") - } - - stat, ok := fileInfo.Sys().(*syscall.Stat_t) - if !ok { - return errors.Errorf("failed to get raw syscall.Stat_t data for '%s'", file) - } - - switch fileInfo.Mode() & os.ModeType { - case os.ModeDir: - if err := createDirIfNotExists(destPath, 0755); err != nil { - return err - } - case os.ModeSymlink: - if err := copySymLink(file, destPath); err != nil { - return err - } - default: - if err := copyFile(file, destPath); err != nil { - return err - } - } - - if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil { - return err - } - - // TODO: when does this happen??? - isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 - if !isSymlink { - if err := os.Chmod(destPath, fileInfo.Mode()); err != nil { - return err - } - } - } - return -} - -// takes a list of paths and adds all files in all subdirectories -func expandPathDirectories(files []string) (out []string, err error) { - for _, file := range files { - if err = filepath.Walk(file, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - out = append(out, path) - return nil - }); err != nil { - return - } - } - return -} - -func copyFile(srcFile, dstFile string) error { - out, err := os.Create(dstFile) - if err != nil { - return err - } - - defer out.Close() - - in, err := os.Open(srcFile) - if err != nil { - return err - } - defer in.Close() - - _, err = io.Copy(out, in) - if err != nil { - return err - } - - return nil -} - -func fileExists(filePath string) bool { - if _, err := os.Stat(filePath); os.IsNotExist(err) { - return false - } - - return true -} - -func createDirIfNotExists(dir string, perm os.FileMode) error { - if fileExists(dir) { - return nil - } - - if err := os.MkdirAll(dir, perm); err != nil { - return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error()) - } - - return nil -} - -func copySymLink(source, dest string) error { - link, err := os.Readlink(source) - if err != nil { - return err - } - return os.Symlink(link, dest) -} diff --git a/pkg/derivation/derivation.go b/pkg/derivation/derivation.go deleted file mode 100644 index d62398ef..00000000 --- a/pkg/derivation/derivation.go +++ /dev/null @@ -1,349 +0,0 @@ -package derivation - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "os" - "os/exec" - "path/filepath" - "sort" - "strings" - - "github.com/maxmcd/bramble/pkg/reptar" - "github.com/maxmcd/bramble/pkg/starutil" - "github.com/maxmcd/bramble/pkg/textreplace" - "github.com/mholt/archiver/v3" - "github.com/pkg/errors" - "go.starlark.net/starlark" -) - -var ( - TempDirPrefix = "bramble-" - PathPaddingCharacters = "bramble_store_padding" - PathPaddingLength = 50 - - // BramblePrefixOfRecord is the prefix we use when hashing the build output - // this allows us to get a consistent hash even if we're building in a - // different location - BramblePrefixOfRecord = "/home/bramble/bramble/bramble_store_padding/bramb" -) - -var ( - ErrStoreDoesNotExist = errors.New("calculated store path doesn't exist, did the location change?") -) - -// Derivation is the basic building block of a Bramble build -type Derivation struct { - Name string - Outputs map[string]Output - Builder string - Platform string - Args []string - Env map[string]string - Sources []string - InputDerivations []InputDerivation - - // internal fields - function *Function - location string -} - -// DerivationOutput tracks the build outputs. Outputs are not included in the -// Derivation hash. The path tracks the output location in the bramble store -// and Dependencies tracks the bramble outputs that are runtime dependencies. -type Output struct { - Path string - Dependencies []string -} - -// InputDerivation is one of the derivation inputs. Path is the location of -// the derivation, output is the name of the specific output this derivation -// uses for the build -type InputDerivation struct { - Path string - Output string -} - -var ( - _ starlark.Value = new(Derivation) - _ starlark.HasAttrs = new(Derivation) -) - -func (drv *Derivation) String() string { - // TODO: we're overriding this for our own purposes. could be confusing - return fmt.Sprintf("", drv.Name) -} - -func (drv *Derivation) Type() string { return "derivation" } -func (drv *Derivation) Freeze() {} -func (drv *Derivation) Truth() starlark.Bool { return starlark.True } -func (drv *Derivation) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("cmd") } - -func (drv *Derivation) Attr(name string) (val starlark.Value, err error) { - output, ok := drv.Outputs[name] - if ok { - return starlark.String(fmt.Sprintf("$bramble_path/%s", output.Path)), nil - } - return nil, nil -} - -func (drv *Derivation) AttrNames() (out []string) { - for name := range drv.Outputs { - out = append(out, name) - } - return -} - -func (drv *Derivation) prettyJSON() string { - b, _ := json.MarshalIndent(drv, "", " ") - return string(b) -} - -func (drv *Derivation) calculateInputDerivations() (err error) { - // TODO: is this the best way to do this? presumaby in nix it's a language - // feature - - fileBytes, err := json.Marshal(drv) - if err != nil { - return - } - for location, derivation := range drv.function.derivations { - // TODO: check all outputs, not just the default - if bytes.Contains(fileBytes, []byte(derivation.String())) { - drv.InputDerivations = append(drv.InputDerivations, InputDerivation{ - Path: location, - Output: "out", - }) - } - } - sort.Slice(drv.InputDerivations, func(i, j int) bool { - id := drv.InputDerivations[i] - jd := drv.InputDerivations[j] - return id.Path+id.Output < jd.Path+id.Output - }) - return nil -} - -func (drv *Derivation) computeDerivation() (fileBytes []byte, filename string, err error) { - fileBytes, err = json.Marshal(drv) - if err != nil { - return - } - outputs := drv.Outputs - // content is hashed without the outputs attribute - drv.Outputs = nil - var jsonBytesForHashing []byte - jsonBytesForHashing, err = json.Marshal(drv) - if err != nil { - return - } - drv.Outputs = outputs - fileName := fmt.Sprintf("%s.drv", drv.Name) - _, filename, err = hashFile(fileName, ioutil.NopCloser(bytes.NewBuffer(jsonBytesForHashing))) - if err != nil { - return - } - return -} - -func (drv *Derivation) checkForExisting() (exists bool, err error) { - _, filename, err := drv.computeDerivation() - if err != nil { - return - } - drv.function.log.Debug("derivation " + drv.Name + " evaluates to " + filename) - existingDrv, exists, err := drv.function.LoadDerivation(filename) - if err != nil { - return false, err - } - if !exists { - return false, nil - } - drv.Outputs = existingDrv.Outputs - return true, nil -} - -func (drv *Derivation) assembleSources(destination string) (runLocation string, err error) { - if len(drv.Sources) == 0 { - return - } - sources := drv.Sources - drv.Sources = []string{} - absDir, err := filepath.Abs(drv.location) - if err != nil { - return - } - // get absolute paths for all sources - for i, src := range sources { - sources[i] = filepath.Join(absDir, src) - } - prefix := commonPrefix(append(sources, absDir)) - - if err = copyFiles(prefix, sources, destination); err != nil { - return - } - relBramblefileLocation, err := filepath.Rel(prefix, absDir) - if err != nil { - return "", errors.Wrap(err, "error calculating relative bramblefile loc") - } - runLocation = filepath.Join(destination, relBramblefileLocation) - if err = os.MkdirAll(runLocation, 0755); err != nil { - return "", errors.Wrap(err, "error making build directory") - } - drv.Env["src"] = destination - return -} - -func (drv *Derivation) writeDerivation() (err error) { - fileBytes, filename, err := drv.computeDerivation() - if err != nil { - return - } - fileLocation := drv.function.joinStorePath(filename) - - if !fileExists(fileLocation) { - return ioutil.WriteFile(fileLocation, fileBytes, 0444) - } - return nil -} - -func (drv *Derivation) createBuildDir() (tempDir string, err error) { - return ioutil.TempDir("", TempDirPrefix) -} - -func (drv *Derivation) computeOutPath() (outPath string, err error) { - _, filename, err := drv.computeDerivation() - - return filepath.Join( - drv.function.storePath, - strings.TrimSuffix(filename, ".drv"), - ), err -} - -func (drv *Derivation) expand(s string) string { - return os.Expand(s, func(i string) string { - if i == "bramble_path" { - return drv.function.storePath - } - if v, ok := drv.Env[i]; ok { - return v - } - return "" - }) -} - -func (drv *Derivation) build() (err error) { - buildDir, err := drv.createBuildDir() - if err != nil { - return - } - - outPath, err := drv.computeOutPath() - if err != nil { - return err - } - if err = os.MkdirAll(outPath, 0755); err != nil { - return - } - if drv.Builder == "fetch_url" { - url, ok := drv.Env["url"] - if !ok { - return errors.New("fetch_url requires the environment variable 'url' to be set") - } - hash, ok := drv.Env["hash"] - if !ok { - return errors.New("fetch_url requires the environment variable 'hash' to be set") - } - path, err := drv.function.DownloadFile(url, hash) - if err != nil { - return err - } - if err = archiver.Unarchive(path, outPath); err != nil { - return errors.Wrap(err, "error unarchiving") - } - } else { - var runLocation string - runLocation, err = drv.assembleSources(buildDir) - if err != nil { - return - } - builderLocation := drv.expand(drv.Builder) - - if _, err := os.Stat(builderLocation); err != nil { - return errors.Wrap(err, "error checking if builder location exists") - } - cmd := exec.Command(builderLocation, drv.Args...) - cmd.Dir = runLocation - cmd.Env = []string{} - for k, v := range drv.Env { - v = strings.Replace(v, "$bramble_path", drv.function.storePath, -1) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) - } - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "out", outPath)) - cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", "bramble_path", drv.function.storePath)) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err = cmd.Run(); err != nil { - return err - } - } - - matches, hashString, err := drv.hashAndScanDirectory(outPath) - if err != nil { - return - } - folderName := hashString + "-" + drv.Name - drv.Outputs["out"] = Output{Path: folderName, Dependencies: matches} - - newPath := drv.function.joinStorePath() + "/" + folderName - _, doesnotExistErr := os.Stat(newPath) - drv.function.log.Debug("Output at ", newPath) - if doesnotExistErr != nil { - return os.Rename(outPath, newPath) - } - // hashed content is already there, just exit - return -} - -func (drv *Derivation) hashAndScanDirectory(location string) (matches []string, hashString string, err error) { - var storeValues []string - old := drv.function.storePath - new := BramblePrefixOfRecord - - for _, derivation := range drv.function.derivations { - storeValues = append(storeValues, strings.Replace(derivation.String(), "$bramble_path", old, 1)) - } - - errChan := make(chan error) - resultChan := make(chan map[string]struct{}) - pipeReader, pipeWriter := io.Pipe() - - go func() { - if err := reptar.Reptar(location, pipeWriter); err != nil { - errChan <- err - } - if err = pipeWriter.Close(); err != nil { - errChan <- err - } - }() - hasher := NewHasher() - go func() { - _, matches, err := textreplace.ReplaceStringsPrefix(pipeReader, hasher, storeValues, old, new) - if err != nil { - errChan <- err - } - resultChan <- matches - }() - select { - case err := <-errChan: - return nil, "", err - case result := <-resultChan: - for k := range result { - matches = append(matches, strings.Replace(k, drv.function.storePath, "$bramble_path", 1)) - } - return matches, hasher.String(), nil - } -} diff --git a/pkg/derivation/function.go b/pkg/derivation/function.go deleted file mode 100644 index be89cbf9..00000000 --- a/pkg/derivation/function.go +++ /dev/null @@ -1,206 +0,0 @@ -package derivation - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - - "github.com/maxmcd/bramble/pkg/starutil" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "go.starlark.net/resolve" - "go.starlark.net/starlark" -) - -// Function is the function that creates derivations -type Function struct { - bramblePath string - storePath string - derivations map[string]*Derivation - thread *starlark.Thread - - log *logrus.Logger -} - -var ( - _ starlark.Value = new(Function) - _ starlark.Callable = new(Function) -) - -func (f *Function) Freeze() {} -func (f *Function) Hash() (uint32, error) { return 0, starutil.ErrUnhashable("module") } -func (f *Function) Name() string { return f.String() } -func (f *Function) String() string { return `` } -func (f *Function) Truth() starlark.Bool { return true } -func (f *Function) Type() string { return "module" } - -func init() { - // It's easier to start giving away free coffee than it is to take away - // free coffee - resolve.AllowFloat = false - resolve.AllowLambda = false - resolve.AllowNestedDef = false - resolve.AllowRecursion = false - resolve.AllowSet = true -} - -// NewFunction creates a new client. When initialized this function checks if the -// bramble store exists and creates it if it does not. -func NewFunction(thread *starlark.Thread) (*Function, error) { - // TODO: don't run on this on every command run, shouldn't be needed to - // just print health information - bramblePath, storePath, err := ensureBramblePath() - if err != nil { - return nil, err - } - // TODO: check that the store directory structure is accurate and make - // directories if needed - fn := &Function{ - log: logrus.New(), - bramblePath: bramblePath, - storePath: storePath, - derivations: make(map[string]*Derivation), - } - // c.log.SetReportCaller(true) - fn.log.SetLevel(logrus.DebugLevel) - - fn.thread = thread - return fn, nil -} - -// Run runs a file given a path. Returns the global variable values from that -// file. Run will recursively run imported files. -func (f *Function) Run(file string) (globals starlark.StringDict, err error) { - f.log.Debug("running file ", file) - globals, err = starlark.ExecFile(f.thread, file, nil, starlark.StringDict{ - "derivation": f, - }) - if err != nil { - return - } - return -} - -func (f *Function) joinStorePath(v ...string) string { - return filepath.Join(append([]string{f.storePath}, v...)...) -} - -// Load derivation will load and parse a derivation from the bramble store1 -func (f *Function) LoadDerivation(filename string) (drv *Derivation, exists bool, err error) { - fileLocation := f.joinStorePath(filename) - _, err = os.Stat(fileLocation) - if err != nil { - return nil, false, nil - } - file, err := os.Open(fileLocation) - if err != nil { - return nil, true, err - } - drv = &Derivation{} - return drv, true, json.NewDecoder(file).Decode(drv) -} - -func (f *Function) buildDerivation(drv *Derivation) (err error) { - var exists bool - exists, err = drv.checkForExisting() - if err != nil { - return - } - if exists { - return - } - if err = drv.build(); err != nil { - return - } - if err = drv.writeDerivation(); err != nil { - return - } - return -} - -// DownloadFile downloads a file into the store. Must include an expected hash -// of the downloaded file as a hex string of a sha256 hash -func (f *Function) DownloadFile(url string, hash string) (path string, err error) { - f.log.Debugf("Downloading url %s", url) - - b, err := hex.DecodeString(hash) - if err != nil { - err = errors.Wrap(err, fmt.Sprintf("error decoding hash %q; is it hexadecimal?", hash)) - return - } - storePrefixHash := bytesToBase32Hash(b) - matches, err := filepath.Glob(f.joinStorePath(storePrefixHash) + "*") - if err != nil { - err = errors.Wrap(err, "error searching for existing hashed content") - return - } - if len(matches) != 0 { - return matches[0], nil - } - resp, err := http.Get(url) - if err != nil { - err = errors.Wrap(err, fmt.Sprintf("error making request to download %q", url)) - return - } - defer resp.Body.Close() - file, err := ioutil.TempFile(filepath.Join(f.bramblePath, "tmp"), "") - if err != nil { - err = errors.Wrap(err, "error creating a temporary file for a download") - return - } - sha256Hash := sha256.New() - tee := io.TeeReader(resp.Body, sha256Hash) - if _, err = io.Copy(file, tee); err != nil { - err = errors.Wrap(err, "error writing to the temporary download file") - return - } - sha256HashBytes := sha256Hash.Sum(nil) - hexStringHash := fmt.Sprintf("%x", sha256HashBytes) - if hash != hexStringHash { - err = errors.Errorf( - "Got incorrect hash for url %s.\nwanted %q\ngot %q", - url, hash, hexStringHash) - // make best effort to save this file, as we'll likely just download it again - storePrefixHash = bytesToBase32Hash(sha256HashBytes) - } - path = f.joinStorePath(storePrefixHash + "-" + filepath.Base(url)) - // don't overwrite err if we error here, we want to try and save this, but - // still return the incorrect hash error - if er := os.Rename(file.Name(), path); er != nil { - return "", errors.Wrap(er, "error moving file into store") - } - return path, err -} - -func (f *Function) CallInternal(thread *starlark.Thread, args starlark.Tuple, kwargs []starlark.Tuple) (v starlark.Value, err error) { - if args.Len() > 0 { - return nil, errors.New("builtin function build() takes no positional arguments") - } - drv, err := f.newDerivationFromKWArgs(kwargs) - if err != nil { - return nil, &starlark.EvalError{Msg: err.Error(), CallStack: f.thread.CallStack()} - } - // At(0) is within this function, we want the file of the caller - drv.location = filepath.Dir(thread.CallStack().At(1).Pos.Filename()) - if err = drv.calculateInputDerivations(); err != nil { - return nil, &starlark.EvalError{Msg: err.Error(), CallStack: f.thread.CallStack()} - } - - f.log.Debugf("Building derivation %q", drv.Name) - if err = f.buildDerivation(drv); err != nil { - return nil, &starlark.EvalError{Msg: err.Error(), CallStack: f.thread.CallStack()} - } - f.log.Debug("Completed derivation: ", drv.prettyJSON()) - _, filename, err := drv.computeDerivation() - if err != nil { - return - } - f.derivations[filename] = drv - return drv, nil -} diff --git a/pkg/derivation/hasher.go b/pkg/derivation/hasher.go deleted file mode 100644 index b4964dbd..00000000 --- a/pkg/derivation/hasher.go +++ /dev/null @@ -1,55 +0,0 @@ -package derivation - -import ( - "bytes" - "crypto/sha256" - "encoding/base32" - "fmt" - "hash" - "io" - "strings" -) - -// Hasher is used to compute path hash values. Hasher implements io.Writer and -// takes a sha256 hash of the input bytes. The output string is a lowercase -// base32 representation of the first 160 bits of the hash -type Hasher struct { - hash hash.Hash -} - -func NewHasher() *Hasher { - return &Hasher{ - hash: sha256.New(), - } -} - -func (h *Hasher) Write(b []byte) (n int, err error) { - return h.hash.Write(b) -} - -func (h *Hasher) String() string { - return bytesToBase32Hash(h.hash.Sum(nil)) -} - -// bytesToBase32Hash copies nix here -// https://nixos.org/nixos/nix-pills/nix-store-paths.html -// Finally the comments tell us to compute the base32 representation of the -// first 160 bits (truncation) of a sha256 of the above string: -func bytesToBase32Hash(b []byte) string { - var buf bytes.Buffer - _, _ = base32.NewEncoder(base32.StdEncoding, &buf).Write(b[:20]) - return strings.ToLower(buf.String()) -} - -func hashFile(name string, file io.ReadCloser) (fileHash, filename string, err error) { - defer file.Close() - hasher := NewHasher() - if _, err = hasher.Write([]byte(name)); err != nil { - return - } - if _, err = io.Copy(hasher, file); err != nil { - return - } - filename = fmt.Sprintf("%s-%s", hasher.String(), name) - return -} diff --git a/pkg/derivation/starlark.go b/pkg/derivation/starlark.go deleted file mode 100644 index 17fe4fc9..00000000 --- a/pkg/derivation/starlark.go +++ /dev/null @@ -1,141 +0,0 @@ -package derivation - -import ( - "fmt" - - "github.com/pkg/errors" - "go.starlark.net/starlark" -) - -type typeError struct { - funcName string - argument string - wantedType string -} - -func (te typeError) Error() string { - return fmt.Sprintf("%s() keyword argument '%s' must be of type '%s'", te.funcName, te.argument, te.wantedType) -} - -func (f *Function) newDerivationFromKWArgs(kwargs []starlark.Tuple) (drv *Derivation, err error) { - te := typeError{ - funcName: "derivation", - } - drv = &Derivation{ - Outputs: map[string]Output{"out": {}}, - Env: map[string]string{}, - function: f, - } - for _, kwarg := range kwargs { - key := kwarg.Index(0).(starlark.String).GoString() - value := kwarg.Index(1) - switch key { - case "name": - name, ok := value.(starlark.String) - if !ok { - te.argument = "name" - te.wantedType = "string" - return drv, te - } - drv.Name = name.GoString() - case "builder": - name, ok := value.(starlark.String) - if !ok { - te.argument = "builder" - te.wantedType = "string" - return drv, te - } - drv.Builder = name.GoString() - case "args": - drv.Args, err = valueToStringArray(value, "derivation", "args") - if err != nil { - return - } - case "sources": - drv.Sources, err = valueToStringArray(value, "derivation", "args") - if err != nil { - return - } - case "env": - drv.Env, err = valueToStringMap(value, "derivation", "env") - if err != nil { - return - } - default: - err = errors.Errorf("derivation() got an unexpected keyword argument '%s'", key) - return - } - } - return drv, nil -} - -func valueToStringArray(val starlark.Value, function, param string) (out []string, err error) { - maybeErr := errors.Errorf( - "%s parameter '%s' expects type 'list' instead got '%s'", - function, param, val.String()) - if val.Type() != "list" { - err = maybeErr - return - } - list, ok := val.(*starlark.List) - if !ok { - err = maybeErr - return - } - for i := 0; i < list.Len(); i++ { - v, ok := list.Index(i).(starlark.String) - if !ok { - err = errors.Errorf("%s %s expects a list of strings, but got value %s", function, param, v.String()) - return - } - out = append(out, v.GoString()) - } - return -} - -func valueToStringMap(val starlark.Value, function, param string) (out map[string]string, err error) { - out = map[string]string{} - maybeErr := errors.Errorf( - "%s parameter '%s' expects type 'dict' instead got '%s'", - function, param, val.String()) - if val.Type() != "dict" { - err = maybeErr - return - } - dict, ok := val.(starlark.IterableMapping) - if !ok { - err = maybeErr - return - } - items := dict.Items() - for _, item := range items { - key := item.Index(0) - value := item.Index(1) - ks, ok := key.(starlark.String) - if !ok { - err = errors.Errorf("%s %s expects a dictionary of strings, but got key '%s'", function, param, key.String()) - return - } - valBool, ok := value.(starlark.Bool) - if ok { - out[ks.GoString()] = "true" - if valBool == starlark.False { - out[ks.GoString()] = "false" - } - continue - } - - drv, ok := value.(*Derivation) - if ok { - out[ks.GoString()] = drv.String() - continue - } - vs, ok := value.(starlark.String) - if !ok { - err = errors.Errorf("%s %s expects a dictionary of strings, but got value '%s'", function, param, value.String()) - return - } - out[ks.GoString()] = vs.GoString() - } - return -} diff --git a/pkg/starutil/conversions.go b/pkg/starutil/conversions.go index d35c6ba1..88a15715 100644 --- a/pkg/starutil/conversions.go +++ b/pkg/starutil/conversions.go @@ -6,11 +6,7 @@ import ( "go.starlark.net/starlark" ) -func ListToListOfStrings(listValue starlark.Value) (out []string, err error) { - list, ok := listValue.(*starlark.List) - if !ok { - return nil, ErrIncorrectType{is: listValue.Type(), shouldBe: "list"} - } +func ListToGoList(list *starlark.List) (out []string, err error) { iterator := list.Iterate() defer iterator.Done() var val starlark.Value @@ -25,12 +21,54 @@ func ListToListOfStrings(listValue starlark.Value) (out []string, err error) { return } +func ListToValueList(list *starlark.List) (out []starlark.Value) { + iterator := list.Iterate() + defer iterator.Done() + var val starlark.Value + for iterator.Next(&val) { + cpy := val + out = append(out, cpy) + } + return +} + +func DictToGoStringMap(dict *starlark.Dict) (out map[string]string, err error) { + out = make(map[string]string) + for _, key := range dict.Keys() { + envVal, _, _ := dict.Get(key) + keyString, err := ValueToString(key) + if err != nil { + return nil, err + } + valString, err := ValueToString(envVal) + if err != nil { + return nil, err + } + out[keyString] = valString + } + return +} + +func ListToListOfStrings(listValue starlark.Value) (out []string, err error) { + list, ok := listValue.(*starlark.List) + if !ok { + return nil, ErrIncorrectType{is: listValue.Type(), shouldBe: "list"} + } + return ListToGoList(list) +} + func ValueToString(val starlark.Value) (out string, err error) { switch v := val.(type) { case starlark.String: out = v.GoString() case starlark.Int: out = v.String() + case starlark.Bool: + if v { + out = "true" + } else { + out = "false" + } default: return "", errors.Errorf("don't know how to cast type %q into a string", v.Type()) } diff --git a/pkg/starutil/derivation_checker.go b/pkg/starutil/derivation_checker.go new file mode 100644 index 00000000..de395da0 --- /dev/null +++ b/pkg/starutil/derivation_checker.go @@ -0,0 +1,4 @@ +package starutil + +type DerivationChecker interface { +} diff --git a/pkg/starutil/errors.go b/pkg/starutil/errors.go index 3342f3bf..3a814900 100644 --- a/pkg/starutil/errors.go +++ b/pkg/starutil/errors.go @@ -29,7 +29,7 @@ func AnnotateError(err error) string { sb := new(strings.Builder) switch err := errors.Cause(err).(type) { case *starlark.EvalError: - if err.CallStack.At(0).Pos.Filename() == "assert.star" { + if len(err.CallStack) > 0 && err.CallStack.At(0).Pos.Filename() == "assert.star" { err.CallStack.Pop() } fmt.Fprintln(sb) diff --git a/readme.md b/readme.md index faba6def..3a2bf52b 100644 --- a/readme.md +++ b/readme.md @@ -2,17 +2,90 @@ Bramble is a functional build system inspired by [nix](https://nixos.org/). -Bramble is currently a work-in-progress. Feel free to read the corresponding blog post for context and background: https://maxmcd.com/posts/lets-build-a-nix-guix +**This codebase is in active development and is not stable, complete or secure, proceed with caution.** -Current Project goals: - - Easy to use and understand - - Run nothing as root - - Provide primitives and tools to create reproducible builds (might with the previous goal) - - First class support for building docker images - - Binary relocation/renaming - - (more to come) +### Overview -## Core Concepts +Bramble is a script runner and build system. You can use it to reliably build libraries and executables and run them on various systems. + +Bramble uses [starlark](https://docs.bazel.build/versions/master/skylark/language.html) as its build language. Starlark's syntax is inspired by python. + +Let's look at a simple example: + +```python +def run_busybox(): + # this call triggers the derivation to calculate and build + bb = busybox() + # whoami is now available to run + print(cmd("whoami", clear_env=True, env={"PATH": bb.out + "/bin"}).output()) + + +def busybox(): + # download the executable, this is the only way you are allowed + # to use the network during builds + download = derivation( + name="busybox_download", + builder="fetch_url", + env={ + "url": "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz", + "hash": "2ae410370b8e9113968ffa6e52f38eea7f17df5f436bd6a69cc41c6ca01541a1", + }, + ) + + # we pass the build_busybox callback here which is used to build this derivation + return derivation(name="busybox", builder=build_busybox, build_inputs=[download]) + + +def build_busybox(): + os.mkdir("$out/bin") # using the builtin os module + + # move the busybox executable into the output for this build + print(cmd("echo ok $busybox_download/busybox-x86_64").output()) + cmd( + "$busybox_download/busybox-x86_64", + "cp", + "$busybox_download/busybox-x86_64", + "$out/bin/busybox", + ).wait() + + # extract the available commands from the busybox help text + commands = [] + commands_started = False + for line in cmd("$out/bin/busybox").output: + if commands_started: + for name in line.strip().split(","): + name = name.strip() + if name: + commands.append(name) + if "Currently defined functions" in line: # the builtin os module + commands_started = True + + # cd into the output directory so that symlinks are relative + os.cd("$out/bin") + cmd("./busybox ln -s busybox ln").wait() + + # link each command so that we can use all the available busybox commands + for c in commands: + if c != "ln": + cmd("./ln -s busybox %s" % c).output() +``` + +We can save this file as "example.bramble" to our filesystem and run it with `bramble run example:run_busybox` + +When we do so the following things happen: + +1. The file is parsed and the global function `run_busybox` is found and run. +2. `busybox()` is called, which triggers two calls to `derivation()` +3. Each derivation is not built, instead a serialized version of the derivation is calculated and held in memory. +4. When `bb.cmd()` is called the script knows that the build stage is complete. Before running the function, all dependent derivations are built. +5. During the build stage the busybox executable is downloaded and then built using the `build_busybox()` function. +6. After the build is complete `bb.cmd("whoami")` is run and the output is printed. + +In order for this all to work there are a few rules: + +- You can't call `cmd()`, `derivation.cmd()`, or use any of the `os` module before derivations are done being called. Calls to the current system might be used as build inputs which would lead to inconsistent behavior in different environments. +- Similarly, using the network within a derivation is disallowed, you must use the `fetch_url` builder to fetch files +- Using `cmd()` and `os` are ok within derivation build functions as these calls will (eventually) be sandboxed. ### Derivations @@ -48,10 +121,27 @@ derivation( ### CLI +Run +```bash +# run the function "foo" in the file default.bramble +bramble run foo + +# run the function foo in the file main.bramble or ./main/default.bramble +bramble run main:foo + +# download the package github.com/maxmcd/bramble and run the function "seed" +# in ./seed/default.bramble +bramble run github.com/maxmcd/bramble/seed:seed ``` -bramble run -``` +Test +```bash +# runs rests in the current directory +bramble test + +# run tests in the ./tests directory +bramble test ./tests +``` ### Store diff --git a/tests/derivation_test.bramble b/tests/derivation_test.bramble index d9123682..21a833a4 100644 --- a/tests/derivation_test.bramble +++ b/tests/derivation_test.bramble @@ -12,4 +12,4 @@ def other(): and_another() def and_another(): - assert.fails(_no_name, "no such") + assert.fails(_no_name, "missing argument for builder") diff --git a/tests/starlark-builder.bramble b/tests/starlark-builder.bramble new file mode 100644 index 00000000..eeac9594 --- /dev/null +++ b/tests/starlark-builder.bramble @@ -0,0 +1,55 @@ +def run_busybox(): + # this call triggers the derivation to calculate and build + bb = busybox() + # whoami is now available to run + print(cmd("whoami", clear_env=True, env={"PATH": bb.out + "/bin"}).output()) + + +def busybox(): + # download the executable, this is the only way you are allowed + # to use the network during builds + download = derivation( + name="busybox_download", + builder="fetch_url", + env={ + "url": "https://brmbl.s3.amazonaws.com/busybox-x86_64.tar.gz", + "hash": "2ae410370b8e9113968ffa6e52f38eea7f17df5f436bd6a69cc41c6ca01541a1", + }, + ) + + # we pass the build_busybox callback here which is used to build this derivation + return derivation(name="busybox", builder=build_busybox, build_inputs=[download]) + + +def build_busybox(): + os.mkdir("$out/bin") # using the builtin os module + + # move the busybox executable into the output for this build + print(cmd("echo ok $busybox_download/busybox-x86_64").output()) + cmd( + "$busybox_download/busybox-x86_64", + "cp", + "$busybox_download/busybox-x86_64", + "$out/bin/busybox", + ).wait() + + # extract the available commands from the busybox help text + commands = [] + commands_started = False + for line in cmd("$out/bin/busybox").output: + if commands_started: + for name in line.strip().split(","): + name = name.strip() + if name: + commands.append(name) + if "Currently defined functions" in line: # the builtin os module + commands_started = True + + # cd into the output directory so that symlinks are relative + os.cd("$out/bin") + cmd("./busybox ln -s busybox ln").wait() + + # link each command so that we can use all the available busybox commands + for c in commands: + if c != "ln": + cmd("./ln -s busybox %s" % c).output()