diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ef9d49c3..b23fb2eb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Go uses: magnetikonline/action-golang-cache@v4 with: - go-version: 1.21.4 + go-version: ~1.22 cache-key-suffix: -build - name: Cache @@ -59,6 +59,12 @@ jobs: echo $VERSION echo "version=$VERSION" >> $GITHUB_OUTPUT +# - name: Setup tmate session +# if: ${{ failure() }} +# run: | +# curl -L https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz | tar -xJ --strip-components 1 +# ./tmate -F + upload_gcs: name: Upload binaries needs: [ build ] @@ -125,7 +131,7 @@ jobs: - name: Set up Go uses: magnetikonline/action-golang-cache@v4 with: - go-version: ~1.21 + go-version: ~1.22 cache-key-suffix: -test - uses: actions/download-artifact@v3 @@ -170,9 +176,3 @@ jobs: - name: Cleanup .heph run: | rm -rf .heph/cache/test - -# - name: Setup tmate session -# if: ${{ failure() }} -# run: | -# curl -L https://github.com/tmate-io/tmate/releases/download/2.4.0/tmate-2.4.0-static-linux-amd64.tar.xz | tar -xJ --strip-components 1 -# ./tmate -F diff --git a/BUILD b/BUILD index 1060f84b..af9ef595 100644 --- a/BUILD +++ b/BUILD @@ -4,7 +4,7 @@ load("//backend/node", "yarn_toolchain") go_toolchain( name = "go", - version = "1.21.4", + version = "1.22.2", architectures = [ "darwin_amd64", "darwin_arm64", diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index d55c5209..931e6e8b 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -8,6 +8,7 @@ import ( "github.com/hephbuild/heph/buildfiles" "github.com/hephbuild/heph/config" "github.com/hephbuild/heph/exprs" + "github.com/hephbuild/heph/gitstatus" "github.com/hephbuild/heph/graph" "github.com/hephbuild/heph/hbuiltin" "github.com/hephbuild/heph/hroot" @@ -23,7 +24,8 @@ import ( "github.com/hephbuild/heph/targetrun" "github.com/hephbuild/heph/upgrade" "github.com/hephbuild/heph/utils/finalizers" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" + "github.com/pbnjay/memory" "os" "path/filepath" "strings" @@ -64,7 +66,7 @@ type BootOpts struct { Summary bool JaegerEndpoint string DisableCloudTelemetry bool - Pool *worker.Pool + Pool *worker2.Engine PostBootBase func(bs BaseBootstrap) error @@ -133,13 +135,22 @@ type Bootstrap struct { Observability *observability.Observability Cloud Cloud Summary *obsummary.Summary - Pool *worker.Pool + Pool *worker2.Engine Packages *packages.Registry BuildFiles *buildfiles.State Graph *graph.State PlatformProviders []platform.PlatformProvider } +func DefaultScheduler(cpu int) *worker2.ResourceScheduler { + return worker2.NewResourceScheduler(map[string]float64{ + "cpu": float64(cpu), + "memory": float64(memory.TotalMemory()), + }, map[string]float64{ + "cpu": float64(1), + }) +} + func Boot(ctx context.Context, opts BootOpts) (Bootstrap, error) { bs := Bootstrap{} @@ -182,7 +193,9 @@ func Boot(ctx context.Context, opts BootOpts) (Bootstrap, error) { pool := opts.Pool if pool == nil { - pool = worker.NewPool(opts.Workers) + pool = worker2.NewEngine() + pool.SetDefaultScheduler(DefaultScheduler(opts.Workers)) + go pool.Run() } bs.Pool = pool @@ -294,21 +307,26 @@ func BootScheduler(ctx context.Context, bs Bootstrap) (*scheduler.Scheduler, err } e := scheduler.New(scheduler.Scheduler{ - Cwd: bs.Cwd, - Root: bs.Root, - Config: bs.Config, - Observability: bs.Observability, - GetFlowID: getFlowId, - LocalCache: localCache, - RemoteCache: remoteCache, - Packages: bs.Packages, - BuildFilesState: bs.BuildFiles, - Graph: bs.Graph, - Pool: bs.Pool, - Finalizers: fins, - Runner: runner, + Cwd: bs.Cwd, + Root: bs.Root, + Config: bs.Config, + Observability: bs.Observability, + GetFlowID: getFlowId, + LocalCache: localCache, + RemoteCache: remoteCache, + Packages: bs.Packages, + BuildFilesState: bs.BuildFiles, + Graph: bs.Graph, + Pool: bs.Pool, + BackgroundTracker: worker2.NewRunningTracker(), + Finalizers: fins, + Runner: runner, }) + if bs.Config.Engine.GitCacheHints { + e.GitStatus = gitstatus.New(bs.Root.Root.Abs()) + } + bs.Finalizers.RegisterWithErr(func(err error) { fins.Run(err) }) diff --git a/bootstrap/config.go b/bootstrap/config.go index 90f4b077..803285f0 100644 --- a/bootstrap/config.go +++ b/bootstrap/config.go @@ -7,6 +7,7 @@ import ( "github.com/hephbuild/heph/hroot" "github.com/hephbuild/heph/log/log" "os" + "time" ) func BuildConfig(root *hroot.State, profiles []string) (*config.Config, error) { @@ -16,6 +17,8 @@ func BuildConfig(root *hroot.State, profiles []string) (*config.Config, error) { cfg.CacheHistory = 3 cfg.Engine.GC = true cfg.Engine.CacheHints = true + cfg.Engine.GitCacheHints = false + cfg.ProgressInterval = time.Second cfg.Engine.ParallelCaching = true cfg.Engine.SmartGen = true cfg.CacheOrder = config.CacheOrderLatency diff --git a/bootstrap/error.go b/bootstrap/error.go index 42c30e47..5ee4bc4f 100644 --- a/bootstrap/error.go +++ b/bootstrap/error.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/hephbuild/heph/log/log" "github.com/hephbuild/heph/targetrun" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "go.uber.org/multierr" "io" "os" @@ -42,7 +42,7 @@ func printErrTargetFailed(err error) bool { } func PrintHumanError(err error) { - errs := multierr.Errors(worker.CollectRootErrors(err)) + errs := worker2.CollectRootErrors(err) skippedCount := 0 skipSpacing := true @@ -57,18 +57,19 @@ func PrintHumanError(err error) { for _, err := range errs { if printErrTargetFailed(err) { // Printed ! + continue + } + + var jerr worker2.Error + if errors.As(err, &jerr) && jerr.Skipped() { + skippedCount++ + skipSpacing = true + log.Debugf("skipped: %v", jerr) } else { - var jerr worker.JobError - if errors.As(err, &jerr) && jerr.Skipped() { - skippedCount++ + for _, err := range multierr.Errors(err) { skipSpacing = true - log.Debugf("skipped: %v", jerr) - } else { - for _, err := range multierr.Errors(err) { - skipSpacing = true - separate() - log.Error(err) - } + separate() + log.Error(err) } } } diff --git a/bootstrap/error_test.go b/bootstrap/error_test.go new file mode 100644 index 00000000..5f607cdf --- /dev/null +++ b/bootstrap/error_test.go @@ -0,0 +1,33 @@ +package bootstrap + +import ( + "errors" + "github.com/hephbuild/heph/worker2" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCollectRootErrors(t *testing.T) { + err := worker2.Error{ + ID: 2, + Name: "a2", + State: worker2.ExecStateSkipped, + Err: worker2.Error{ + ID: 1, + Name: "a1", + State: worker2.ExecStateFailed, + Err: errors.New("sad beep bop"), + }, + } + + errs := worker2.CollectRootErrors(err) + + assert.EqualValues(t, []error{ + worker2.Error{ + ID: 1, + Name: "a1", + State: worker2.ExecStateFailed, + Err: errors.New("sad beep bop"), + }, + }, errs) +} diff --git a/bootstrap/rrs.go b/bootstrap/rrs.go index a9489db2..966b3523 100644 --- a/bootstrap/rrs.go +++ b/bootstrap/rrs.go @@ -11,7 +11,7 @@ import ( "github.com/hephbuild/heph/targetrun" "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/sets" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2/poolwait" ) var errHasExprDep = errors.New("has expr, bailing out") @@ -125,7 +125,7 @@ func RunGen(ctx context.Context, e *scheduler.Scheduler, plain bool, filterFacto return err } - err = poolwait.Wait(ctx, fmt.Sprintf("Gen run %v", i), e.Pool, deps, plain) + err = poolwait.Wait(ctx, fmt.Sprintf("Gen run %v", i), e.Pool, deps, plain, e.Config.ProgressInterval) if err != nil { return err } diff --git a/bootstrap/run.go b/bootstrap/run.go index 1e8587d5..c8fdd0de 100644 --- a/bootstrap/run.go +++ b/bootstrap/run.go @@ -4,13 +4,13 @@ import ( "context" "errors" "fmt" + "github.com/dlsniper/debugger" "github.com/hephbuild/heph/log/log" "github.com/hephbuild/heph/sandbox" "github.com/hephbuild/heph/scheduler" "github.com/hephbuild/heph/specs" "github.com/hephbuild/heph/targetrun" - "github.com/hephbuild/heph/worker" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2/poolwait" "os" "os/exec" ) @@ -37,6 +37,12 @@ func Run(ctx context.Context, e *scheduler.Scheduler, rrs targetrun.Requests, ru } func RunMode(ctx context.Context, e *scheduler.Scheduler, rrs targetrun.Requests, runopts RunOpts, inlineSingle bool, mode string, iocfg sandbox.IOConfig) error { + debugger.SetLabels(func() []string { + return []string{ + "where", "RunMode", + } + }) + for i := range rrs { rrs[i].Mode = mode } @@ -60,32 +66,19 @@ func RunMode(ctx context.Context, e *scheduler.Scheduler, rrs targetrun.Requests inlineRR = &rrs[0] } - // fgDeps will include deps created inside the scheduled jobs to be waited for in the foreground - // The DoneSem() must be called after all the tdeps have finished - ctx, fgDeps := poolwait.ContextWithForegroundWaitGroup(ctx) - fgDeps.AddSem() - var skip []specs.Specer if inlineRR != nil { skip = []specs.Specer{inlineRR.Target} } - tdepsMap, err := e.ScheduleTargetRRsWithDeps(ctx, rrs, skip) + tdepsMap, tracker, err := e.ScheduleTargetRRsWithDeps(ctx, rrs, skip) if err != nil { - fgDeps.DoneSem() return err } tdeps := tdepsMap.All() - go func() { - <-tdeps.Done() - fgDeps.DoneSem() - }() - - runDeps := &worker.WaitGroup{} - runDeps.AddChild(tdeps) - runDeps.AddChild(fgDeps) + tdeps.AddDep(tracker.Group()) - err = poolwait.Wait(ctx, "Run", e.Pool, runDeps, runopts.Plain) + err = poolwait.Wait(ctx, "Run", e.Pool, tdeps, runopts.Plain, e.Config.ProgressInterval) if err != nil { return err } @@ -119,7 +112,7 @@ func RunMode(ctx context.Context, e *scheduler.Scheduler, rrs targetrun.Requests iocfg.Stdout = os.Stderr } - err = e.RunWithSpan(ctx, *inlineRR, iocfg) + err = e.RunWithSpan(ctx, *inlineRR, iocfg, nil) if err != nil { var eerr *exec.ExitError if errors.As(err, &eerr) { diff --git a/bootstrapwatch/bootstrap.go b/bootstrapwatch/bootstrap.go index 0bfc2243..7b086179 100644 --- a/bootstrapwatch/bootstrap.go +++ b/bootstrapwatch/bootstrap.go @@ -17,8 +17,8 @@ import ( "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/maps" "github.com/hephbuild/heph/utils/xfs" - "github.com/hephbuild/heph/worker" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2" + "github.com/hephbuild/heph/worker2/poolwait" "io/fs" "os" "path/filepath" @@ -40,7 +40,7 @@ type State struct { rropts targetrun.RequestOpts cbs bootstrap.SchedulerBootstrap cbbs bootstrap.BaseBootstrap - pool *worker.Pool + pool *worker2.Engine sigCh chan sigEvent triggeredHashed maps.Map[string, string] @@ -88,7 +88,9 @@ func Boot(ctx context.Context, root *hroot.State, bootopts bootstrap.BootOpts, c return nil, err } - pool := worker.NewPool(bootopts.Workers) + pool := worker2.NewEngine() + pool.SetDefaultScheduler(bootstrap.DefaultScheduler(bootopts.Workers)) + go pool.Run() bootopts.Pool = pool bbs, err := bootstrap.BootBase(ctx, bootopts) @@ -117,7 +119,6 @@ func Boot(ctx context.Context, root *hroot.State, bootopts bootstrap.BootOpts, c log.Error("watcher close:", err) } close(sigCh) - pool.Stop(nil) }, } @@ -349,6 +350,10 @@ func (s *State) trigger(ctx context.Context, events []fsEvent) error { status("Figuring out if anything changed...") printEvents(events) + if bs.Scheduler.GitStatus != nil { + bs.Scheduler.GitStatus.Reset() + } + rrs, err := bootstrap.GenerateRRs(ctx, bs.Scheduler, s.matcher, s.targs, s.rropts, s.runopts.Plain, true) if err != nil { return err @@ -365,14 +370,16 @@ func (s *State) trigger(ctx context.Context, events []fsEvent) error { } // Run the rrs's deps, excluding the rrs's themselves - tdepsMap, err := bs.Scheduler.ScheduleTargetRRsWithDeps(ctx, rrs, specs.AsSpecers(rrs.Targets().Slice())) + tdepsMap, tracker, err := bs.Scheduler.ScheduleTargetRRsWithDeps(ctx, rrs, specs.AsSpecers(rrs.Targets().Slice())) if err != nil { return err } - tdeps := tdepsMap.All() + runDeps := worker2.NewNamedGroup("trigger rundeps") + runDeps.AddDep(tdepsMap.All()) + runDeps.AddDep(tracker.Group()) - err = poolwait.Wait(ctx, "Change", bs.Pool, tdeps, s.runopts.Plain) + err = poolwait.Wait(ctx, "Change", bs.Pool, runDeps, s.runopts.Plain, bs.Config.ProgressInterval) if err != nil { return err } diff --git a/cmd/heph/init.go b/cmd/heph/init.go index cb041082..3ba4fed4 100644 --- a/cmd/heph/init.go +++ b/cmd/heph/init.go @@ -59,22 +59,17 @@ func schedulerInit(ctx context.Context, postBoot func(bootstrap.BaseBootstrap) e Finalizers.RegisterWithErr(func(err error) { bs.Finalizers.Run(err) - if !bs.Scheduler.Pool.IsDone() { + gb := bs.Scheduler.BackgroundTracker.Group() + bs.Pool.Schedule(gb) + if !gb.GetState().IsFinal() { log.Tracef("Waiting for all pool items to finish") select { - case <-bs.Scheduler.Pool.Done(): + case <-gb.Wait(): case <-time.After(time.Second): log.Infof("Waiting for background jobs to finish...") - <-bs.Scheduler.Pool.Done() + <-gb.Wait() } log.Tracef("All pool items finished") - - bs.Scheduler.Pool.Stop(nil) - - err := bs.Scheduler.Pool.Err() - if err != nil { - log.Error(err) - } } if bs.Summary != nil { @@ -107,8 +102,6 @@ func bootstrapInit(ctx context.Context) (bootstrap.Bootstrap, error) { Finalizers.RegisterWithErr(func(err error) { bs.Finalizers.Run(err) - - bs.Pool.Stop(err) }) return bs, nil diff --git a/cmd/heph/query.go b/cmd/heph/query.go index 38e93e69..3a2515db 100644 --- a/cmd/heph/query.go +++ b/cmd/heph/query.go @@ -18,7 +18,7 @@ import ( "github.com/hephbuild/heph/targetrun" "github.com/hephbuild/heph/utils/sets" "github.com/hephbuild/heph/utils/xfs" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2/poolwait" "github.com/itchyny/gojq" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -642,7 +642,7 @@ var revdepsCmd = &cobra.Command{ return fmt.Errorf("%v is outside repo", p) } - children := bs.Graph.DAG().GetFileChildren([]string{rel}, bs.Graph.Targets().Slice()) + children := bs.Graph.DAG().GetFileChildren([]string{p}, bs.Graph.Targets().Slice()) if err != nil { return err } @@ -804,12 +804,12 @@ var hashinCmd = &cobra.Command{ return err } - tdeps, err := bs.Scheduler.ScheduleTargetsWithDeps(ctx, []*graph.Target{target}, false, []specs.Specer{target}) + tdeps, _, err := bs.Scheduler.ScheduleTargetsWithDeps(ctx, []*graph.Target{target}, false, []specs.Specer{target}) if err != nil { return err } - err = poolwait.Wait(ctx, "Run", bs.Scheduler.Pool, tdeps.All(), *plain) + err = poolwait.Wait(ctx, "Run", bs.Scheduler.Pool, tdeps.All(), *plain, bs.Config.ProgressInterval) if err != nil { return err } diff --git a/config/config.go b/config/config.go index 1bf5dcab..77345470 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "gopkg.in/yaml.v3" "os" "path/filepath" + "time" ) type BaseConfig struct { @@ -21,17 +22,19 @@ const ( ) type Config struct { - BaseConfig `yaml:",inline"` - Caches map[string]Cache - CacheOrder string `yaml:"cache_order"` - CacheHistory int `yaml:"cache_history"` - Cloud struct { + BaseConfig `yaml:",inline"` + Caches map[string]Cache + CacheOrder string `yaml:"cache_order"` + CacheHistory int `yaml:"cache_history"` + ProgressInterval time.Duration `yaml:"progress_interval"` + Cloud struct { URL string `yaml:"url"` Project string `yaml:"project"` } `yaml:"cloud"` Engine struct { GC bool `yaml:"gc"` CacheHints bool `yaml:"cache_hints"` + GitCacheHints bool `yaml:"git_cache_hints"` InstallTools bool `yaml:"install_tools"` KeepSandbox bool `yaml:"keep_sandbox"` ParallelCaching bool `yaml:"parallel_caching"` diff --git a/config/file_config.go b/config/file_config.go index bf92ca6b..c733050a 100644 --- a/config/file_config.go +++ b/config/file_config.go @@ -1,19 +1,23 @@ package config +import "time" + type Extras map[string]interface{} type FileConfig struct { - BaseConfig `yaml:",inline"` - Caches map[string]FileCache `yaml:"caches,omitempty"` - CacheOrder string `yaml:"cache_order"` - CacheHistory int `yaml:"cache_history"` - Cloud struct { + BaseConfig `yaml:",inline"` + Caches map[string]FileCache `yaml:"caches,omitempty"` + CacheOrder string `yaml:"cache_order"` + CacheHistory int `yaml:"cache_history"` + ProgressInterval *int `yaml:"progress_interval"` + Cloud struct { URL string `yaml:"url"` Project string `yaml:"project"` } `yaml:"cloud"` Engine struct { GC *bool `yaml:"gc"` CacheHints *bool `yaml:"cache_hints"` + GitCacheHints *bool `yaml:"git_cache_hints"` InstallTools *bool `yaml:"install_tools"` KeepSandbox *bool `yaml:"keep_sandbox"` ParallelCaching *bool `yaml:"parallel_caching"` @@ -77,10 +81,18 @@ func (fc FileConfig) ApplyTo(c Config) Config { c.Engine.CacheHints = *fc.Engine.CacheHints } + if fc.Engine.GitCacheHints != nil { + c.Engine.GitCacheHints = *fc.Engine.GitCacheHints + } + if fc.Engine.InstallTools != nil { c.Engine.InstallTools = *fc.Engine.InstallTools } + if fc.ProgressInterval != nil { + c.ProgressInterval = time.Duration(*fc.ProgressInterval) * time.Second + } + if fc.CacheOrder != "" { c.CacheOrder = fc.CacheOrder } diff --git a/gitstatus/gitstatus.go b/gitstatus/gitstatus.go new file mode 100644 index 00000000..1d183dad --- /dev/null +++ b/gitstatus/gitstatus.go @@ -0,0 +1,70 @@ +package gitstatus + +import ( + "context" + "github.com/hephbuild/heph/utils/ads" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +type GitStatus struct { + root string + o sync.Once + + dirty map[string]struct{} +} + +func New(root string) *GitStatus { + return &GitStatus{root: root} +} + +func (gs *GitStatus) diffIndexOnce(ctx context.Context) map[string]struct{} { + gs.o.Do(func() { + // If something went wrong, we can assume nothing is dirty + dirty, _ := gs.diffIndex(ctx) + + gs.dirty = make(map[string]struct{}, len(dirty)) + for _, file := range dirty { + gs.dirty[file] = struct{}{} + } + }) + + return gs.dirty +} + +func (gs *GitStatus) diffIndex(ctx context.Context) ([]string, error) { + gitPath, err := exec.LookPath("git") + if err != nil { + return nil, err + } + + cmd := exec.CommandContext(ctx, gitPath, "diff-index", "HEAD", "--name-only") + cmd.Dir = gs.root + + b, err := cmd.Output() + if err != nil { + return nil, err + } + + lines := strings.Split(string(b), "\n") + lines = ads.Filter(lines, func(s string) bool { + return s != "" + }) + + return ads.Map(lines, func(line string) string { + return filepath.Join(gs.root, line) + }), nil +} + +func (gs *GitStatus) IsDirty(ctx context.Context, path string) bool { + dirty := gs.diffIndexOnce(ctx) + + _, ok := dirty[path] + return ok +} + +func (gs *GitStatus) Reset() { + gs.o = sync.Once{} +} diff --git a/gitstatus/gitstatus_test.go b/gitstatus/gitstatus_test.go new file mode 100644 index 00000000..0e29a2f1 --- /dev/null +++ b/gitstatus/gitstatus_test.go @@ -0,0 +1,63 @@ +package gitstatus + +import ( + "context" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestIsDirty(t *testing.T) { + t.SkipNow() // fails in CI, run locally + + dir, err := os.MkdirTemp("", "") + require.NoError(t, err) + + defer os.RemoveAll(dir) + + ctx := context.Background() + gs := New(dir) + + git := func(t *testing.T, args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = dir + err = cmd.Run() + require.NoError(t, err) + } + + write := func(t *testing.T, file, content string) { + err = os.WriteFile(filepath.Join(dir, file), []byte(content), os.ModePerm) + require.NoError(t, err) + gs.Reset() + } + + write(t, "file1", "a") + write(t, "file2", "a") + write(t, "file3", "a") + + git(t, "init") + git(t, "add", ".") + git(t, "commit", "-m", "initial") + + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file1"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file2"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file3"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "doesnt_exist"))) + + write(t, "file2", "b") + + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file1"))) + assert.True(t, gs.IsDirty(ctx, filepath.Join(dir, "file2"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file3"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "doesnt_exist"))) + + git(t, "add", ".") + + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file1"))) + assert.True(t, gs.IsDirty(ctx, filepath.Join(dir, "file2"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "file3"))) + assert.False(t, gs.IsDirty(ctx, filepath.Join(dir, "doesnt_exist"))) +} diff --git a/go.mod b/go.mod index 42dd647d..9e993c1e 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.21 replace github.com/spf13/cobra v1.7.0 => github.com/raphaelvigee/cobra v0.0.0-20221020122344-217ca52feee0 require ( + cloud.google.com/go/storage v1.34.1 github.com/Khan/genqlient v0.6.0 github.com/aarondl/json v0.0.0-20221020222930-8b0db17ef1bf github.com/aarondl/opt v0.0.0-20230313190023-85d93d668fec @@ -14,10 +15,11 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/c2fo/vfs/v6 v6.9.0 github.com/charmbracelet/bubbles v0.16.1 - github.com/charmbracelet/bubbletea v0.24.2 + github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-semver v0.3.1 github.com/creack/pty v1.1.20 + github.com/dlsniper/debugger v0.6.0 github.com/fsnotify/fsnotify v1.7.0 github.com/google/uuid v1.4.0 github.com/heimdalr/dag v1.3.1 @@ -29,6 +31,7 @@ require ( github.com/muesli/reflow v0.3.0 github.com/muesli/termenv v0.15.2 github.com/olekukonko/tablewriter v0.0.5 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/shirou/gopsutil/v3 v3.23.10 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -43,8 +46,10 @@ require ( go.starlark.net v0.0.0-20231101134539-556fd59b42f6 go.uber.org/multierr v1.11.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/net v0.17.0 golang.org/x/sys v0.14.0 golang.org/x/term v0.13.0 + google.golang.org/api v0.149.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -53,7 +58,6 @@ require ( cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.5 // indirect - cloud.google.com/go/storage v1.34.1 // indirect github.com/RoaringBitmap/roaring v1.6.0 // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alexflint/go-arg v1.4.3 // indirect @@ -123,13 +127,11 @@ require ( go.opentelemetry.io/otel/metric v1.19.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sync v0.5.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.149.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231030173426-d783a09b4405 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20231030173426-d783a09b4405 // indirect diff --git a/go.sum b/go.sum index 02a67b2c..8a9adb4b 100644 --- a/go.sum +++ b/go.sum @@ -83,8 +83,8 @@ github.com/c2fo/vfs/v6 v6.9.0/go.mod h1:AKtJfE/GU0bX5xYeNSrPJaWHNyUUuHLsMpi6zw8F github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= -github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= -github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= +github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -102,6 +102,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/dlsniper/debugger v0.6.0 h1:AyPoOtJviCmig9AKNRAPPw5B5UyB+cI72zY3Jb+6LlA= +github.com/dlsniper/debugger v0.6.0/go.mod h1:FFdRcPU2Yo4P411bp5U97DHJUSUMKcqw1QMGUu0uVb8= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -240,6 +242,8 @@ github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/graph/artifacts_factory.go b/graph/artifacts_factory.go index c39803c4..67ccbdb6 100644 --- a/graph/artifacts_factory.go +++ b/graph/artifacts_factory.go @@ -55,8 +55,17 @@ func (o *ArtifactRegistry) OutHash(name string) artifacts.Artifact { return o.Out[name].Hash() } +func (o *ArtifactRegistry) OutTar2(name string) (artifacts.Artifact, bool) { + out, ok := o.Out[name] + if !ok { + return nil, false + } + return out.Tar(), true +} + func (o *ArtifactRegistry) OutTar(name string) artifacts.Artifact { - return o.Out[name].Tar() + a, _ := o.OutTar2(name) + return a } func (e *State) newArtifactRegistry(target *Target) *ArtifactRegistry { diff --git a/graph/dag.go b/graph/dag.go index 02931e35..636f44b4 100644 --- a/graph/dag.go +++ b/graph/dag.go @@ -193,13 +193,9 @@ func (d *DAG) GetFileChildren(paths []string, universe []*Target) []*Target { descendants := NewTargets(0) for _, path := range paths { - for _, target := range universe { - for _, file := range target.HashDeps.Files { - if file.RelRoot() == path { - descendants.Add(target) - break - } - } + target := d.getFileChildren(path, universe) + if target != nil { + descendants.Add(target) } } @@ -208,6 +204,24 @@ func (d *DAG) GetFileChildren(paths []string, universe []*Target) []*Target { return descendants.Slice() } +func (d *DAG) getFileChildren(path string, universe []*Target) *Target { + for _, target := range universe { + for _, file := range target.Deps.All().Files { + if file.Abs() == path { + return target + } + } + for _, file := range target.HashDeps.Files { + if file.Abs() == path { + return target + + } + } + } + + return nil +} + func (d *DAG) mapToArray(m map[string]interface{}) []*Target { a := make([]*Target, 0, len(m)) for _, anci := range m { diff --git a/hbuiltin/predeclared.go b/hbuiltin/predeclared.go index 8f16adb9..fc4cda8d 100644 --- a/hbuiltin/predeclared.go +++ b/hbuiltin/predeclared.go @@ -14,6 +14,7 @@ import ( "github.com/hephbuild/heph/utils/xfs" "github.com/hephbuild/heph/utils/xstarlark" "github.com/hephbuild/heph/utils/xsync" + "github.com/pbnjay/memory" starlarkjson "go.starlark.net/lib/json" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" @@ -79,6 +80,8 @@ func predeclared_functions() starlark.StringDict { "param": starlark.NewBuiltin("heph.param", param), "cache": starlark.NewBuiltin("heph.cache", starlarkstruct.Make), "target_spec": starlark.NewBuiltin("heph.target_spec", starlarkstruct.Make), + "num_cpu": starlark.NewBuiltin("heph.num_cpu", num_cpu), + "total_memory": starlark.NewBuiltin("heph.total_memory", total_memory), //"normalize_target_name": starlark.NewBuiltin("heph.normalize_target_name", normalize_target_name), //"normalize_pkg_name": starlark.NewBuiltin("heph.normalize_target_name", normalize_pkg_name), "pkg": &starlarkstruct.Module{ @@ -180,6 +183,7 @@ func target(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, "timeout?", &sargs.Timeout, "gen_deps_meta?", &sargs.GenDepsMeta, "annotations?", &sargs.Annotations, + "requests?", &sargs.Requests, ); err != nil { if sargs.Name != "" { return nil, fmt.Errorf("%v: %w", pkg.TargetAddr(sargs.Name), err) @@ -391,6 +395,14 @@ func param(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, k return starlark.String(cfg.Params[name]), nil } +func num_cpu(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + return starlark.MakeInt(runtime.NumCPU()), nil +} + +func total_memory(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + return starlark.MakeUint64(memory.TotalMemory()), nil +} + func path_base(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var ( path string diff --git a/hbuiltin/predeclared_types.go b/hbuiltin/predeclared_types.go index e2e7fa5d..a2cadc11 100644 --- a/hbuiltin/predeclared_types.go +++ b/hbuiltin/predeclared_types.go @@ -41,6 +41,7 @@ type TargetArgs struct { Timeout string GenDepsMeta bool Annotations xstarlark.Distruct + Requests xstarlark.Distruct } type TargetArgsTransitive struct { diff --git a/hbuiltin/register_target_test.go b/hbuiltin/register_target_test.go index ef4641a5..0f336e7c 100644 --- a/hbuiltin/register_target_test.go +++ b/hbuiltin/register_target_test.go @@ -12,6 +12,7 @@ import ( "github.com/hephbuild/heph/utils/xfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.starlark.net/starlark" "io" "io/fs" "os" @@ -168,3 +169,25 @@ func TestDocFromArgs(t *testing.T) { }) } } + +func TestParseResource(t *testing.T) { + tests := []struct { + v starlark.Value + expected float64 + }{ + {starlark.MakeInt(1), 1}, + {starlark.Float(1.2), 1.2}, + {starlark.String("1"), 1}, + {starlark.String("100m"), 0.1}, + {starlark.String("100k"), 100000}, + {starlark.String("100Ki"), 102400}, + } + for _, test := range tests { + t.Run(test.v.String(), func(t *testing.T) { + actual, err := requestFromArg(test.v) + require.NoError(t, err) + + assert.Equal(t, test.expected, actual) + }) + } +} diff --git a/hbuiltin/target_spec_parser.go b/hbuiltin/target_spec_parser.go index 5cdad26c..ddb0b959 100644 --- a/hbuiltin/target_spec_parser.go +++ b/hbuiltin/target_spec_parser.go @@ -11,6 +11,7 @@ import ( "github.com/hephbuild/heph/utils/sets" "github.com/hephbuild/heph/utils/xstarlark" "go.starlark.net/starlark" + "math" "runtime" "strconv" "strings" @@ -194,6 +195,19 @@ func specFromArgs(args TargetArgs, pkg *packages.Package) (specs.Target, error) t.Annotations[item.Key] = utils.FromStarlark(item.Value) } + t.Requests = map[string]float64{ + "cpu": 1, + "memory": 100000000, // 1M + } + for _, item := range args.Requests.Items() { + v, err := requestFromArg(item.Value) + if err != nil { + return specs.Target{}, fmt.Errorf("request: %v: %w", item.Key, err) + } + + t.Requests[item.Key] = v + } + if t.OutEnv == "" { if t.OutInSandbox { t.OutEnv = specs.FileEnvAbs @@ -248,6 +262,53 @@ func specFromArgs(args TargetArgs, pkg *packages.Package) (specs.Target, error) return t, nil } +// https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/ +var units = map[string]float64{ + "E": math.Pow(10, 18), + "P": math.Pow(10, 15), + "T": math.Pow(10, 12), + "G": math.Pow(10, 9), + "M": math.Pow(10, 6), + "k": math.Pow(10, 3), + + "Ei": math.Pow(2, 60), + "Pi": math.Pow(2, 50), + "Ti": math.Pow(2, 40), + "Gi": math.Pow(2, 30), + "Mi": math.Pow(2, 20), + "Ki": math.Pow(2, 10), + + "m": math.Pow(10, -3), +} + +func requestFromArg(s starlark.Value) (float64, error) { + switch v := utils.FromStarlark(s).(type) { + case string: + mul := float64(1) + for u, uv := range units { + rest, ok := strings.CutSuffix(v, u) + if ok { + mul = uv + v = rest + break + } + } + + vi, err := strconv.ParseFloat(v, 10) + if err != nil { + return 0, err + } + + return vi * mul, nil + case int64: + return float64(v), nil + case float64: + return v, nil + default: + return 0, fmt.Errorf("unsupported type %T: %v", v, v) + } +} + func docFromArg(doc string) string { if len(doc) == 0 { return "" diff --git a/hbuiltin/testdata/cache-false b/hbuiltin/testdata/cache-false index 92a9752a..5378a7b5 100644 --- a/hbuiltin/testdata/cache-false +++ b/hbuiltin/testdata/cache-false @@ -114,5 +114,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/cache-history b/hbuiltin/testdata/cache-history index b9d6ab04..9357c416 100644 --- a/hbuiltin/testdata/cache-history +++ b/hbuiltin/testdata/cache-history @@ -114,5 +114,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/cache-named b/hbuiltin/testdata/cache-named index f187cb40..93c2ed41 100644 --- a/hbuiltin/testdata/cache-named +++ b/hbuiltin/testdata/cache-named @@ -118,5 +118,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/cache-named-empty b/hbuiltin/testdata/cache-named-empty index 7e3a407f..35e5067f 100644 --- a/hbuiltin/testdata/cache-named-empty +++ b/hbuiltin/testdata/cache-named-empty @@ -116,5 +116,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/cache-named-false b/hbuiltin/testdata/cache-named-false index 58e81235..00d04377 100644 --- a/hbuiltin/testdata/cache-named-false +++ b/hbuiltin/testdata/cache-named-false @@ -116,5 +116,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/cache-true b/hbuiltin/testdata/cache-true index 80f366a6..950ecfd8 100644 --- a/hbuiltin/testdata/cache-true +++ b/hbuiltin/testdata/cache-true @@ -114,5 +114,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/hbuiltin/testdata/sanity b/hbuiltin/testdata/sanity index fe1e2a9b..e5abeb25 100644 --- a/hbuiltin/testdata/sanity +++ b/hbuiltin/testdata/sanity @@ -139,5 +139,9 @@ target( "Timeout": 0, "GenDepsMeta": false, "Annotations": {}, + "Requests": { + "cpu": 1, + "memory": 100000000 + }, "Private": false } \ No newline at end of file diff --git a/lcache/artifacts_gen.go b/lcache/artifacts_gen.go index e9f0f6fa..0cd67af0 100644 --- a/lcache/artifacts_gen.go +++ b/lcache/artifacts_gen.go @@ -14,11 +14,11 @@ import ( "github.com/hephbuild/heph/tgt" "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/locks" + "github.com/hephbuild/heph/utils/xdebug" "github.com/hephbuild/heph/utils/xfs" "github.com/hephbuild/heph/utils/xio" "github.com/hephbuild/heph/utils/xmath" - "github.com/hephbuild/heph/worker" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2" "io" "math" "math/rand" @@ -136,23 +136,23 @@ func (e *LocalCacheState) LockArtifacts(ctx context.Context, target graph.Target // For hashing to work properly: // - each hash must be preceded by its tar // - support_files must be first then the other artifacts -func (e *LocalCacheState) createDepsGenArtifacts(target *graph.Target, artsp []ArtifactWithProducer) (map[string]*worker.WaitGroup, map[string]*worker.WaitGroup) { +func (e *LocalCacheState) createDepsGenArtifacts(ctx context.Context, target *graph.Target, artsp []ArtifactWithProducer) (map[string]*worker2.Group, map[string]*worker2.Sem) { arts := target.Artifacts - deps := map[string]*worker.WaitGroup{} + deps := map[string]*worker2.Group{} for _, artifact := range artsp { - wg := &worker.WaitGroup{} + wg := worker2.NewNamedGroup(xdebug.Sprintf("%v: artifact deps: %v", target.Name, artifact.Name())) deps[artifact.Name()] = wg } - signals := map[string]*worker.WaitGroup{} + signals := map[string]*worker2.Sem{} for _, artifact := range artsp { - wg := &worker.WaitGroup{} - wg.AddSem() + wg := worker2.NewSemDep(ctx, xdebug.Sprintf("%v: artifact signal: %v", target.Name, artifact.Name())) + wg.AddSem(1) signals[artifact.Name()] = wg } - var support *worker.WaitGroup + var support worker2.Dep if target.HasSupportFiles { support = signals[arts.OutHash(specs.SupportFilesOutput).Name()] } @@ -161,30 +161,30 @@ func (e *LocalCacheState) createDepsGenArtifacts(target *graph.Target, artsp []A tname := arts.OutTar(output).Name() hname := arts.OutHash(output).Name() - deps[hname].AddChild(signals[tname]) + deps[hname].AddDep(signals[tname]) if support != nil && output != specs.SupportFilesOutput { - deps[tname].AddChild(support) + deps[tname].AddDep(support) } } meta := []string{arts.InputHash.Name(), arts.Manifest.Name()} - allButMeta := &worker.WaitGroup{} + allButMeta := worker2.NewNamedGroup(xdebug.Sprintf("%v: all but meta", target.Addr)) for _, art := range artsp { if !ads.Contains(meta, art.Name()) { - allButMeta.AddChild(signals[art.Name()]) + allButMeta.AddDep(signals[art.Name()]) } } for _, name := range meta { - deps[name].AddChild(allButMeta) + deps[name].AddDep(allButMeta) } return deps, signals } -func (e *LocalCacheState) ScheduleGenArtifacts(ctx context.Context, gtarget graph.Targeter, arts []ArtifactWithProducer, compress bool) (_ *worker.WaitGroup, rerr error) { +func (e *LocalCacheState) ScheduleGenArtifacts(ctx context.Context, gtarget graph.Targeter, arts []ArtifactWithProducer, compress bool) (*worker2.Group, error) { target := gtarget.GraphTarget() dirp, err := e.cacheDir(target) @@ -204,25 +204,28 @@ func (e *LocalCacheState) ScheduleGenArtifacts(ctx context.Context, gtarget grap return nil, err } - allDeps := &worker.WaitGroup{} + allDeps := worker2.NewNamedGroup("gen artifacts alldeps") - deps, signals := e.createDepsGenArtifacts(target, arts) + deps, signals := e.createDepsGenArtifacts(ctx, target, arts) for _, artifact := range arts { artifact := artifact shouldCompress := artifact.Compressible() && compress - j := e.Pool.Schedule(ctx, &worker.Job{ + j := e.Pool.Schedule(worker2.NewAction(worker2.ActionConfig{ + Ctx: ctx, Name: fmt.Sprintf("store %v|%v", target.Addr, artifact.Name()), - Deps: deps[artifact.Name()], - Hook: worker.StageHook{ - OnEnd: func(job *worker.Job) context.Context { - signals[artifact.Name()].DoneSem() - return nil - }, + Deps: []worker2.Dep{deps[artifact.Name()]}, + Hooks: []worker2.Hook{ + worker2.StageHook{ + OnEnd: func(worker2.Dep) context.Context { + signals[artifact.Name()].DoneSem() + return nil + }, + }.Hook(), }, - Do: func(w *worker.Worker, ctx context.Context) error { + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { err := GenArtifact(ctx, dir, artifact, shouldCompress, func(percent float64) { var s string if target.Cache.Enabled { @@ -241,14 +244,13 @@ func (e *LocalCacheState) ScheduleGenArtifacts(ctx context.Context, gtarget grap return nil }, - }) - allDeps.Add(j) - } + })) - if fgDeps := poolwait.ForegroundWaitGroup(ctx); fgDeps != nil { - fgDeps.AddChild(allDeps) + allDeps.AddDep(j) } + e.Pool.Schedule(allDeps) + return allDeps, nil } diff --git a/lcache/lcache.go b/lcache/lcache.go index 9f22a990..40cf4e47 100644 --- a/lcache/lcache.go +++ b/lcache/lcache.go @@ -15,6 +15,7 @@ import ( "github.com/hephbuild/heph/specs" "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/tgt" + "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/finalizers" "github.com/hephbuild/heph/utils/locks" "github.com/hephbuild/heph/utils/maps" @@ -23,13 +24,12 @@ import ( "github.com/hephbuild/heph/utils/xfs" "github.com/hephbuild/heph/utils/xmath" "github.com/hephbuild/heph/vfssimple" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "io" "io/fs" "math" "os" "path/filepath" - "strings" "sync" ) @@ -43,12 +43,12 @@ type LocalCacheState struct { Finalizers *finalizers.Finalizers EnableGC bool ParallelCaching bool - Pool *worker.Pool + Pool *worker2.Engine } const LatestDir = "latest" -func NewState(root *hroot.State, pool *worker.Pool, targets *graph.Targets, obs *observability.Observability, finalizers *finalizers.Finalizers, gc, parallelCaching bool) (*LocalCacheState, error) { +func NewState(root *hroot.State, pool *worker2.Engine, targets *graph.Targets, obs *observability.Observability, finalizers *finalizers.Finalizers, gc, parallelCaching bool) (*LocalCacheState, error) { cachePath := root.Home.Join("cache") loc, err := vfssimple.NewLocation("file://" + cachePath.Abs() + "/") if err != nil { @@ -133,7 +133,7 @@ func (e *LocalCacheState) StoreCache(ctx context.Context, ttarget graph.Targeter return err } - err = worker.SuspendWaitGroup(ctx, genDeps) + err = worker2.WaitDep(ctx, genDeps) if err != nil { return err } @@ -392,25 +392,37 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou } outDir := cacheDir.Join("_output") + + // Legacy... outDirHashPath := cacheDir.Join("_output_hash").Abs() + _ = os.Remove(outDirHashPath) - // TODO: This can be a problem, where 2 targets depends on the same target, but with different outputs, - // leading to the expand overriding each other + outDirMetaPath := cacheDir.Join("_output_meta").Abs() - outDirHash := "2|" + strings.Join(outputs, ",") + type OutDirMeta struct { + Version int + Outputs []string + } + version := 1 shouldExpand := false + shouldCleanExpand := false if !xfs.PathExists(outDir.Abs()) { shouldExpand = true } else { - b, err := os.ReadFile(outDirHashPath) + b, err := os.ReadFile(outDirMetaPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { - return outDir, fmt.Errorf("outdirhash: %w", err) + return outDir, fmt.Errorf("get outdir meta: %w", err) } - if len(b) > 0 && strings.TrimSpace(string(b)) != outDirHash { + var currentMeta OutDirMeta + _ = json.Unmarshal(b, ¤tMeta) + if currentMeta.Version != version || !ads.ContainsAll(currentMeta.Outputs, outputs) { shouldExpand = true } + if currentMeta.Version != version { + shouldCleanExpand = true + } } if len(outputs) == 0 { @@ -419,18 +431,14 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou if shouldExpand { status.Emit(ctx, tgt.TargetStatus(target, "Expanding cache...")) - dir, err := e.cacheDir(target) - if err != nil { - return xfs.Path{}, err - } - tmpOutDir := dir.Join("_output_tmp").Abs() - - err = os.RemoveAll(tmpOutDir) - if err != nil { - return outDir, err + if shouldCleanExpand { + err = os.RemoveAll(outDir.Abs()) + if err != nil { + return outDir, err + } } - err = os.MkdirAll(tmpOutDir, os.ModePerm) + err = os.MkdirAll(outDir.Abs(), os.ModePerm) if err != nil { return outDir, err } @@ -449,6 +457,7 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou if err != nil { return outDir, err } + defer r.Close() var progress func(written int64) if manifest.Size > 0 { @@ -464,30 +473,29 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou return xfs.Path{}, err } - err = tar.UntarContext(ctx, r, tmpOutDir, tar.UntarOptions{ + err = tar.UntarContext(ctx, r, outDir.Abs(), tar.UntarOptions{ ListPath: tarPath, Dedup: untarDedup, Progress: progress, }) - _ = r.Close() if err != nil { return outDir, fmt.Errorf("%v: untar: %w", name, err) } - } - err = os.RemoveAll(outDir.Abs()) - if err != nil { - return outDir, err + _ = r.Close() } - err = os.Rename(tmpOutDir, outDir.Abs()) + b, err := json.Marshal(OutDirMeta{ + Version: version, + Outputs: outputs, + }) if err != nil { - return outDir, err + return xfs.Path{}, err } - err = os.WriteFile(outDirHashPath, []byte(outDirHash), os.ModePerm) + err = os.WriteFile(outDirMetaPath, b, os.ModePerm) if err != nil { - return outDir, fmt.Errorf("outdirhash: %w", err) + return outDir, fmt.Errorf("write outdir meta: %w", err) } } diff --git a/observability/worker.go b/observability/worker.go index f55f3c9b..ec98a56b 100644 --- a/observability/worker.go +++ b/observability/worker.go @@ -2,10 +2,10 @@ package observability import ( "context" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" ) -type createSpanFunc[S any] func(job *worker.Job) (context.Context, S) +type createSpanFunc[S any] func(job worker2.Dep) (context.Context, S) type workerStageStore[S any] struct { span S @@ -13,7 +13,7 @@ type workerStageStore[S any] struct { createSpanFunc createSpanFunc[S] } -func (s *workerStageStore[S]) createSpan(job *worker.Job) (context.Context, S) { +func (s *workerStageStore[S]) createSpan(job worker2.Dep) (context.Context, S) { ctx, span := s.createSpanFunc(job) s.span = span s.hasSpan = true @@ -21,32 +21,32 @@ func (s *workerStageStore[S]) createSpan(job *worker.Job) (context.Context, S) { return ctx, span } -func WorkerStageFactory[S SpanError](f createSpanFunc[S]) worker.Hook { +func WorkerStageFactory[S SpanError](f createSpanFunc[S]) worker2.Hook { ss := &workerStageStore[S]{createSpanFunc: f} - return worker.StageHook{ - OnScheduled: func(job *worker.Job) context.Context { + return worker2.StageHook{ + OnScheduled: func(job worker2.Dep) context.Context { ctx, span := ss.createSpan(job) - span.SetScheduledTime(job.TimeScheduled) + span.SetScheduledTime(job.GetScheduledAt()) return ctx }, - OnQueued: func(job *worker.Job) context.Context { - ss.span.SetQueuedTime(job.TimeQueued) + OnQueued: func(job worker2.Dep) context.Context { + ss.span.SetQueuedTime(job.GetQueuedAt()) return nil }, - OnStart: func(job *worker.Job) context.Context { + OnStart: func(job worker2.Dep) context.Context { var ctx context.Context if !ss.hasSpan { ctx, _ = ss.createSpan(job) } - ss.span.SetStartTime(job.TimeStart) + ss.span.SetStartTime(job.GetStartedAt()) return ctx }, - OnEnd: func(job *worker.Job) context.Context { - if err := job.Err(); err != nil { + OnEnd: func(job worker2.Dep) context.Context { + if err := job.GetErr(); err != nil { state := StateFailed - if job.State == worker.StateSkipped { + if job.GetState() == worker2.ExecStateSkipped { state = StateSkipped } ss.span.EndErrorState(err, state) @@ -55,5 +55,5 @@ func WorkerStageFactory[S SpanError](f createSpanFunc[S]) worker.Hook { } return nil }, - } + }.Hook() } diff --git a/rcache/cache_selection.go b/rcache/cache_selection.go index 505d39e2..2f4134ac 100644 --- a/rcache/cache_selection.go +++ b/rcache/cache_selection.go @@ -62,6 +62,10 @@ func (c orderCacheContainer) calculateLatency(ctx context.Context) (time.Duratio } func orderCaches(ctx context.Context, caches []CacheConfig) []CacheConfig { + caches = ads.Filter(caches, func(cc CacheConfig) bool { + return cc.Read || cc.Write + }) + if len(caches) <= 1 { return caches } diff --git a/scheduler/cache.go b/scheduler/cache.go index c8eb38ac..a63e2f3c 100644 --- a/scheduler/cache.go +++ b/scheduler/cache.go @@ -9,9 +9,12 @@ import ( "github.com/hephbuild/heph/lcache" "github.com/hephbuild/heph/log/log" "github.com/hephbuild/heph/rcache" + "github.com/hephbuild/heph/specs" "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/tgt" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/utils/mds" + "github.com/hephbuild/heph/utils/xdebug" + "github.com/hephbuild/heph/worker2" "os" "sync" ) @@ -69,6 +72,19 @@ func (e *Scheduler) pullOrGetCache(ctx context.Context, target *graph.Target, ou return false, true, nil } + if e.GitStatus != nil { + for _, file := range target.Deps.All().Files { + if e.GitStatus.IsDirty(ctx, file.Abs()) { + err = e.setCacheHintSkip(target, mds.Keys(e.Config.Caches)) + if err != nil { + log.Error(fmt.Errorf("set cache hint: %w", err)) + } + log.Tracef("%v: %v is dirty, skipping cache get", target.Addr, file.Abs()) + return false, false, nil + } + } + } + orderedCaches, err := e.RemoteCache.OrderedCaches(ctx) if err != nil { return false, false, fmt.Errorf("orderedcaches: %w", err) @@ -113,13 +129,9 @@ func (e *Scheduler) pullOrGetCache(ctx context.Context, target *graph.Target, ou log.Warnf("%v cache %v: local cache is supposed to exist locally, but failed getLocalCache, this is not supposed to happen", target.Addr, cache.Name) } else { if e.Config.Engine.CacheHints { - children, err := e.Graph.DAG().GetDescendants(target.Target) + err = e.setCacheHintSkip(target, []string{cache.Name}) if err != nil { - log.Error(fmt.Errorf("descendants: %w", err)) - } - - for _, child := range children { - e.RemoteCache.Hints.Set(child.Addr, cache.Name, rcache.HintSkip{}) + log.Error(fmt.Errorf("set cache hint: %w", err)) } } } @@ -128,6 +140,25 @@ func (e *Scheduler) pullOrGetCache(ctx context.Context, target *graph.Target, ou return false, false, nil } +func (e *Scheduler) setCacheHintSkip(target specs.Specer, cacheNames []string) error { + for _, cacheName := range cacheNames { + e.RemoteCache.Hints.Set(target.Spec().Addr, cacheName, rcache.HintSkip{}) + } + + children, err := e.Graph.DAG().GetDescendants(target) + if err != nil { + return fmt.Errorf("descendants: %w", err) + } + + for _, child := range children { + for _, cacheName := range cacheNames { + e.RemoteCache.Hints.Set(child.Addr, cacheName, rcache.HintSkip{}) + } + } + + return nil +} + func (e *Scheduler) pullExternalCache(ctx context.Context, target *graph.Target, outputs []string, onlyMeta bool, cache rcache.CacheConfig) (_ bool, rerr error) { ctx, span := e.Observability.SpanExternalCacheGet(ctx, target.GraphTarget(), cache.Name, outputs, onlyMeta) defer rcache.SpanEndIgnoreNotExist(span, rerr) @@ -171,29 +202,36 @@ func (e *Scheduler) pullExternalCache(ctx context.Context, target *graph.Target, return true, nil } -func (e *Scheduler) scheduleStoreExternalCache(ctx context.Context, target *graph.Target, cache rcache.CacheConfig) *worker.Job { +func (e *Scheduler) scheduleStoreExternalCache(ctx context.Context, target *graph.Target, cache rcache.CacheConfig, trackers []*worker2.RunningTracker) worker2.Dep { // input hash is used as a marker that everything went well, // wait for everything else to be done before copying the input hash inputHashArtifact := target.Artifacts.InputHash - deps := &worker.WaitGroup{} + deps := worker2.NewNamedGroup(xdebug.Sprintf("%v: schedule store external cache", target.Name)) for _, artifact := range target.Artifacts.All() { if artifact.Name() == inputHashArtifact.Name() { continue } - j := e.scheduleStoreExternalCacheArtifact(ctx, target, cache, artifact, nil) - deps.Add(j) + j := e.scheduleStoreExternalCacheArtifact(ctx, target, cache, artifact, nil, trackers) + deps.AddDep(j) } - return e.scheduleStoreExternalCacheArtifact(ctx, target, cache, inputHashArtifact, deps) + return e.scheduleStoreExternalCacheArtifact(ctx, target, cache, inputHashArtifact, deps, trackers) } -func (e *Scheduler) scheduleStoreExternalCacheArtifact(ctx context.Context, target *graph.Target, cache rcache.CacheConfig, artifact artifacts.Artifact, deps *worker.WaitGroup) *worker.Job { - return e.Pool.Schedule(ctx, &worker.Job{ - Name: fmt.Sprintf("cache %v %v %v", target.Addr, cache.Name, artifact.Name()), - Deps: deps, - Do: func(w *worker.Worker, ctx context.Context) error { +func (e *Scheduler) scheduleStoreExternalCacheArtifact(ctx context.Context, target *graph.Target, cache rcache.CacheConfig, artifact artifacts.Artifact, deps *worker2.Group, trackers []*worker2.RunningTracker) worker2.Dep { + var hooks []worker2.Hook + for _, tracker := range trackers { + hooks = append(hooks, tracker.Hook()) + } + + return e.Pool.Schedule(worker2.NewAction(worker2.ActionConfig{ + Name: fmt.Sprintf("cache %v %v %v", target.Addr, cache.Name, artifact.Name()), + Hooks: hooks, + Deps: []worker2.Dep{deps}, + Ctx: ctx, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { exists, err := e.LocalCache.ArtifactExists(ctx, target, artifact) if err != nil { return err @@ -214,5 +252,5 @@ func (e *Scheduler) scheduleStoreExternalCacheArtifact(ctx context.Context, targ return nil }, - }) + })) } diff --git a/scheduler/engine.go b/scheduler/engine.go index ae51f2c6..5acdc4db 100644 --- a/scheduler/engine.go +++ b/scheduler/engine.go @@ -4,6 +4,7 @@ import ( "context" "github.com/hephbuild/heph/buildfiles" "github.com/hephbuild/heph/config" + "github.com/hephbuild/heph/gitstatus" "github.com/hephbuild/heph/graph" "github.com/hephbuild/heph/hroot" "github.com/hephbuild/heph/lcache" @@ -15,25 +16,29 @@ import ( "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/finalizers" "github.com/hephbuild/heph/utils/locks" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/utils/xdebug" + "github.com/hephbuild/heph/worker2" + "golang.org/x/exp/maps" "sync" ) type Scheduler struct { - Cwd string - Root *hroot.State - Config *config.Config - Observability *observability.Observability - GetFlowID func() string - LocalCache *lcache.LocalCacheState - RemoteCache *rcache.RemoteCache - RemoteCacheHints *rcache.HintStore - Packages *packages.Registry - BuildFilesState *buildfiles.State - Graph *graph.State - Pool *worker.Pool - Finalizers *finalizers.Finalizers - Runner *targetrun.Runner + Cwd string + Root *hroot.State + Config *config.Config + Observability *observability.Observability + GetFlowID func() string + LocalCache *lcache.LocalCacheState + RemoteCache *rcache.RemoteCache + RemoteCacheHints *rcache.HintStore + Packages *packages.Registry + BuildFilesState *buildfiles.State + Graph *graph.State + Pool *worker2.Engine + BackgroundTracker *worker2.RunningTracker + Finalizers *finalizers.Finalizers + Runner *targetrun.Runner + GitStatus *gitstatus.GitStatus toolsLock locks.Locker } @@ -45,23 +50,17 @@ func New(e Scheduler) *Scheduler { type WaitGroupMap struct { mu sync.Mutex - m map[string]*worker.WaitGroup + m map[string]worker2.Dep } -func (wgm *WaitGroupMap) All() *worker.WaitGroup { +func (wgm *WaitGroupMap) All() worker2.Dep { wgm.mu.Lock() defer wgm.mu.Unlock() - wg := &worker.WaitGroup{} - - for _, e := range wgm.m { - wg.AddChild(e) - } - - return wg + return worker2.NewGroup(maps.Values(wgm.m)...) } -func (wgm *WaitGroupMap) Get(s string) *worker.WaitGroup { +func (wgm *WaitGroupMap) Get(s string) worker2.Dep { wgm.mu.Lock() defer wgm.mu.Unlock() @@ -70,16 +69,20 @@ func (wgm *WaitGroupMap) Get(s string) *worker.WaitGroup { } if wgm.m == nil { - wgm.m = map[string]*worker.WaitGroup{} + wgm.m = map[string]worker2.Dep{} } - wg := &worker.WaitGroup{} + wg := worker2.NewNamedGroup(xdebug.Sprintf("groupmap: get: %v", s)) wgm.m[s] = wg return wg } -func (e *Scheduler) ScheduleTargetsWithDeps(ctx context.Context, targets []*graph.Target, pullCache bool, skip []specs.Specer) (*WaitGroupMap, error) { +func (wgm *WaitGroupMap) List() []worker2.Dep { + return maps.Values(wgm.m) +} + +func (e *Scheduler) ScheduleTargetsWithDeps(ctx context.Context, targets []*graph.Target, pullCache bool, skip []specs.Specer) (*WaitGroupMap, *worker2.RunningTracker, error) { rrs := ads.Map(targets, func(t *graph.Target) targetrun.Request { return targetrun.Request{Target: t, RequestOpts: targetrun.RequestOpts{PullCache: pullCache}} }) diff --git a/scheduler/gen_run.go b/scheduler/gen_run.go index ea8eaed8..2ecb3773 100644 --- a/scheduler/gen_run.go +++ b/scheduler/gen_run.go @@ -12,63 +12,61 @@ import ( "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/xfs" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "path/filepath" "time" ) type runGenScheduler struct { - Name string - deps *worker.WaitGroup + tracker *worker2.RunningTracker *Scheduler } -func (e *Scheduler) ScheduleGenPass(ctx context.Context, genTargets []*graph.Target) (_ *worker.WaitGroup, rerr error) { +func (e *Scheduler) ScheduleGenPass(ctx context.Context, genTargets []*graph.Target) (_ worker2.Dep, rerr error) { if len(genTargets) == 0 { log.Debugf("No gen targets, skip gen pass") - return &worker.WaitGroup{}, nil + return worker2.NewNamedGroup("schedule gen pass: empty"), nil } - ctx, span := e.Observability.SpanGenPass(ctx) - defer func() { - if rerr != nil { - span.EndError(rerr) - } - }() - log.Debugf("Run gen pass") ge := runGenScheduler{ - Name: "Main", Scheduler: e, - deps: &worker.WaitGroup{}, + tracker: worker2.NewRunningTracker(), } - err := ge.ScheduleGeneratedPipeline(ctx, genTargets) - if err != nil { - return nil, err - } + ctx, span := e.Observability.SpanGenPass(ctx) + + j1 := worker2.NewAction(worker2.ActionConfig{ + Name: "schedule gen pipeline", + Ctx: ctx, + Hooks: []worker2.Hook{ge.tracker.Hook()}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { + return ge.ScheduleGeneratedPipeline(ctx, genTargets) + }, + }) - j := e.Pool.Schedule(ctx, &worker.Job{ + j := worker2.NewAction(worker2.ActionConfig{ Name: "finalize gen", - Deps: ge.deps, - Hook: worker.StageHook{ - OnEnd: func(job *worker.Job) context.Context { - span.EndError(job.Err()) - return nil - }, + Ctx: ctx, + Hooks: []worker2.Hook{ + worker2.StageHook{ + OnEnd: func(dep worker2.Dep) context.Context { + span.EndError(dep.GetErr()) + return nil + }, + }.Hook(), }, - Do: func(w *worker.Worker, ctx context.Context) error { + Deps: []worker2.Dep{j1, ge.tracker.Group()}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { status.Emit(ctx, status.String("Finalizing gen...")) return nil }, }) - deps := worker.WaitGroupJob(j) - - return deps, nil + return j, nil } func (e *runGenScheduler) ScheduleGeneratedPipeline(ctx context.Context, targets []*graph.Target) error { @@ -78,6 +76,8 @@ func (e *runGenScheduler) ScheduleGeneratedPipeline(ctx context.Context, targets } } + status.Emit(ctx, status.String(fmt.Sprintf("Linking targets..."))) + err := e.Graph.LinkTargets(ctx, false, targets, false) if err != nil { return fmt.Errorf("linking: %w", err) @@ -85,24 +85,28 @@ func (e *runGenScheduler) ScheduleGeneratedPipeline(ctx context.Context, targets start := time.Now() - sdeps, err := e.ScheduleTargetsWithDeps(ctx, targets, true, nil) + status.Emit(ctx, status.String(fmt.Sprintf("Scheduling..."))) + + sdeps, _, err := e.ScheduleTargetsWithDeps(ctx, targets, true, nil) if err != nil { return err } newTargets := graph.NewTargets(0) - deps := &worker.WaitGroup{} + deps := worker2.NewNamedGroup("schedule generated pipeline deps") for _, target := range targets { e.scheduleRunGenerated(ctx, target, sdeps.Get(target.Addr), deps, newTargets) } - j := e.Pool.Schedule(ctx, &worker.Job{ - Name: "ScheduleGeneratedPipeline " + e.Name, - Deps: deps, - Do: func(w *worker.Worker, ctx context.Context) error { - status.Emit(ctx, status.String(fmt.Sprintf("Finalizing generated %v...", e.Name))) + e.Pool.Schedule(worker2.NewAction(worker2.ActionConfig{ + Name: "ScheduleGeneratedPipeline Main", + Ctx: ctx, + Hooks: []worker2.Hook{e.tracker.Hook()}, + Deps: []worker2.Dep{deps}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { + status.Emit(ctx, status.String(fmt.Sprintf("Finalizing generated..."))) - log.Tracef("run generated %v got %v targets in %v", e.Name, newTargets.Len(), time.Since(start)) + log.Tracef("run generated got %v targets in %v", newTargets.Len(), time.Since(start)) genTargets := ads.Filter(newTargets.Slice(), func(target *graph.Target) bool { return target.IsGen() @@ -117,23 +121,23 @@ func (e *runGenScheduler) ScheduleGeneratedPipeline(ctx context.Context, targets return nil }, - }) - e.deps.Add(j) + })) return nil } -func (e *runGenScheduler) scheduleRunGenerated(ctx context.Context, target *graph.Target, runDeps *worker.WaitGroup, deps *worker.WaitGroup, targets *graph.Targets) { - j := e.Pool.Schedule(ctx, &worker.Job{ +func (e *runGenScheduler) scheduleRunGenerated(ctx context.Context, target *graph.Target, runDeps worker2.Dep, deps *worker2.Group, targets *graph.Targets) { + j := worker2.NewAction(worker2.ActionConfig{ Name: "rungen_" + target.Addr, - Deps: runDeps, - Do: func(w *worker.Worker, ctx context.Context) error { + Deps: []worker2.Dep{runDeps}, + Ctx: ctx, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { ltarget := e.LocalCache.Metas.Find(target) return e.scheduleRunGeneratedFiles(ctx, ltarget, deps, targets) }, }) - deps.Add(j) + deps.AddDep(j) } type matchGen struct { @@ -141,7 +145,7 @@ type matchGen struct { matchers []specs.Matcher } -func (e *runGenScheduler) scheduleRunGeneratedFiles(ctx context.Context, target *lcache.Target, deps *worker.WaitGroup, targets *graph.Targets) error { +func (e *runGenScheduler) scheduleRunGeneratedFiles(ctx context.Context, target *lcache.Target, deps *worker2.Group, targets *graph.Targets) error { matchers := []matchGen{{ addr: target.Addr, matchers: target.Gen, @@ -158,78 +162,75 @@ func (e *runGenScheduler) scheduleRunGeneratedFiles(ctx context.Context, target files := target.ActualOutFiles().All().WithRoot(target.OutExpansionRoot().Abs()) - chunks := ads.Chunk(files, len(e.Pool.Workers)) - - for i, files := range chunks { - files := files - - j := e.Pool.Schedule(ctx, &worker.Job{ - Name: fmt.Sprintf("rungen %v chunk %v", target.Addr, i), - Do: func(w *worker.Worker, ctx context.Context) error { - opts := hbuiltin.Bootstrap(hbuiltin.Opts{ - Pkgs: e.Packages, - Root: e.Root, - Config: e.Config, - RegisterTarget: func(spec specs.Target) error { - for _, entry := range matchers { - addrMatchers := ads.Filter(entry.matchers, func(m specs.Matcher) bool { - return specs.IsAddrMatcher(m) - }) - - addrMatch := ads.Some(addrMatchers, func(m specs.Matcher) bool { - return m.Match(spec) - }) - - if !addrMatch { - return fmt.Errorf("%v doest match any gen pattern of %v: %v", spec.Addr, entry.addr, entry.matchers) - } - - labelMatchers := ads.Filter(entry.matchers, func(m specs.Matcher) bool { - return specs.IsLabelMatcher(m) - }) - - for _, label := range spec.Labels { - labelMatch := ads.Some(labelMatchers, func(m specs.Matcher) bool { - return m.(specs.StringMatcher).MatchString(label) - }) - - if !labelMatch { - return fmt.Errorf("label `%v` doest match any gen pattern of %v: %v", label, entry.addr, entry.matchers) - } - } - } - - spec.GenSources = []string{target.Addr} - - t, err := e.Graph.Register(spec) - if err != nil { - return err - } - - targets.Add(t) - return nil - }, + opts := hbuiltin.Bootstrap(hbuiltin.Opts{ + Pkgs: e.Packages, + Root: e.Root, + Config: e.Config, + RegisterTarget: func(spec specs.Target) error { + for _, entry := range matchers { + addrMatchers := ads.Filter(entry.matchers, func(m specs.Matcher) bool { + return specs.IsAddrMatcher(m) }) - for _, file := range files { - status.Emit(ctx, status.String(fmt.Sprintf("Running %v", file.RelRoot()))) + addrMatch := ads.Some(addrMatchers, func(m specs.Matcher) bool { + return m.Match(spec) + }) - ppath := filepath.Dir(file.RelRoot()) - pkg := e.Packages.GetOrCreate(packages.Package{ - Path: ppath, - Root: xfs.NewPath(e.Root.Root.Abs(), ppath), + if !addrMatch { + return fmt.Errorf("%v doest match any gen pattern of %v: %v", spec.Addr, entry.addr, entry.matchers) + } + + labelMatchers := ads.Filter(entry.matchers, func(m specs.Matcher) bool { + return specs.IsLabelMatcher(m) + }) + + for _, label := range spec.Labels { + labelMatch := ads.Some(labelMatchers, func(m specs.Matcher) bool { + return m.(specs.StringMatcher).MatchString(label) }) - err := e.BuildFilesState.RunBuildFile(pkg, file.Abs(), opts) - if err != nil { - return fmt.Errorf("runbuild %v: %w", file.Abs(), err) + if !labelMatch { + return fmt.Errorf("label `%v` doest match any gen pattern of %v: %v", label, entry.addr, entry.matchers) } } + } + + spec.GenSources = []string{target.Addr} + + t, err := e.Graph.Register(spec) + if err != nil { + return err + } + + targets.Add(t) + return nil + }, + }) + + for _, file := range files { + file := file + j := worker2.NewAction(worker2.ActionConfig{ + Name: fmt.Sprintf("rungen %v file %v", target.Addr, file.RelRoot()), + Ctx: ctx, + Hooks: []worker2.Hook{e.tracker.Hook()}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { + status.Emit(ctx, status.String(fmt.Sprintf("Running %v", file.RelRoot()))) + + ppath := filepath.Dir(file.RelRoot()) + pkg := e.Packages.GetOrCreate(packages.Package{ + Path: ppath, + Root: xfs.NewPath(e.Root.Root.Abs(), ppath), + }) + + err := e.BuildFilesState.RunBuildFile(pkg, file.Abs(), opts) + if err != nil { + return fmt.Errorf("runbuild %v: %w", file.Abs(), err) + } return nil }, }) - deps.Add(j) + deps.AddDep(j) } return nil diff --git a/scheduler/schedulerv2.go b/scheduler/schedulerv2.go index 45a100c2..c20d904d 100644 --- a/scheduler/schedulerv2.go +++ b/scheduler/schedulerv2.go @@ -13,35 +13,41 @@ import ( "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/maps" "github.com/hephbuild/heph/utils/sets" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/utils/xdebug" + "github.com/hephbuild/heph/worker2" ) -func (e *Scheduler) ScheduleTargetRun(ctx context.Context, rr targetrun.Request, deps *worker.WaitGroup) (*worker.Job, error) { - j := e.Pool.Schedule(ctx, &worker.Job{ +func (e *schedulerv2) ScheduleTargetRun(ctx context.Context, rr targetrun.Request, deps worker2.Dep) (worker2.Dep, error) { + j := worker2.NewAction(worker2.ActionConfig{ Name: rr.Target.Addr, - Deps: deps, - Hook: observability.WorkerStageFactory(func(job *worker.Job) (context.Context, *observability.TargetSpan) { - return e.Observability.SpanRun(job.Ctx(), rr.Target.GraphTarget()) - }), - Do: func(w *worker.Worker, ctx context.Context) error { - err := e.Run(ctx, rr, sandbox.IOConfig{}) + Deps: []worker2.Dep{deps}, + Ctx: ctx, + Hooks: []worker2.Hook{ + e.tracker.Hook(), + observability.WorkerStageFactory(func(job worker2.Dep) (context.Context, *observability.TargetSpan) { + return e.Observability.SpanRun(job.GetCtx(), rr.Target.GraphTarget()) + }), + }, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { + err := e.Run(ctx, rr, sandbox.IOConfig{}, e.tracker) if err != nil { return targetrun.WrapTargetFailed(err, rr.Target) } return nil }, + Requests: rr.Target.Requests, }) return j, nil } -func (e *Scheduler) ScheduleTargetRRsWithDeps(octx context.Context, rrs targetrun.Requests, skip []specs.Specer) (_ *WaitGroupMap, rerr error) { +func (e *Scheduler) ScheduleTargetRRsWithDeps(octx context.Context, rrs targetrun.Requests, skip []specs.Specer) (*WaitGroupMap, *worker2.RunningTracker, error) { targetsSet := rrs.Targets() toAssess, outputs, err := e.Graph.DAG().GetOrderedAncestorsWithOutput(targetsSet, true) if err != nil { - return nil, err + return nil, nil, err } for _, target := range targetsSet.Slice() { @@ -60,22 +66,24 @@ func (e *Scheduler) ScheduleTargetRRsWithDeps(octx context.Context, rrs targetru }), rrTargets: targetsSet, + tracker: worker2.NewRunningTracker(), + toAssess: toAssess, outputs: outputs, deps: &WaitGroupMap{}, pullMetaDeps: &WaitGroupMap{}, targetSchedLock: &maps.KMutex{}, - targetSchedJobs: &maps.Map[string, *worker.Job]{}, - getCacheOrRunSchedJobs: &maps.Map[getCacheOrRunRequest, *worker.WaitGroup]{}, + targetSchedJobs: &maps.Map[string, worker2.Dep]{}, + getCacheOrRunSchedJobs: &maps.Map[getCacheOrRunRequest, worker2.Dep]{}, } err = sched.schedule() if err != nil { - return nil, err + return nil, nil, err } - return sched.deps, nil + return sched.deps, sched.tracker, nil } type getCacheOrRunRequest struct { @@ -85,6 +93,8 @@ type getCacheOrRunRequest struct { type schedulerv2 struct { *Scheduler + tracker *worker2.RunningTracker + octx context.Context sctx context.Context rrs targetrun.Requests @@ -96,41 +106,47 @@ type schedulerv2 struct { deps *WaitGroupMap pullMetaDeps *WaitGroupMap targetSchedLock *maps.KMutex - targetSchedJobs *maps.Map[string, *worker.Job] - getCacheOrRunSchedJobs *maps.Map[getCacheOrRunRequest, *worker.WaitGroup] + targetSchedJobs *maps.Map[string, worker2.Dep] + getCacheOrRunSchedJobs *maps.Map[getCacheOrRunRequest, worker2.Dep] } func (s *schedulerv2) schedule() error { + for _, target := range s.toAssess.Slice() { + targetDeps := s.deps.Get(target.Addr) + + parents, err := s.Graph.DAG().GetParents(target) + if err != nil { + return err + } + + for _, parent := range parents { + targetDeps.AddDep(s.deps.Get(parent.Addr)) + } + } + for _, target := range s.toAssess.Slice() { target := target targetDeps := s.deps.Get(target.Addr) - targetDeps.AddSem() - - s.pullMetaDeps.Get(target.Addr).AddSem() parents, err := s.Graph.DAG().GetParents(target) if err != nil { return err } - pmdeps := &worker.WaitGroup{} + pmdeps := worker2.NewNamedGroup(xdebug.Sprintf("pmdeps %v", target.Name)) for _, parent := range parents { - pmdeps.AddChild(s.pullMetaDeps.Get(parent.Addr)) + pmdeps.AddDep(s.pullMetaDeps.Get(parent.Addr)) } isSkip := ads.Contains(s.skip, target.Addr) - pj := s.Pool.Schedule(s.sctx, &worker.Job{ - Name: "pull_meta " + target.Addr, - Deps: pmdeps, - Hook: worker.StageHook{ - OnEnd: func(job *worker.Job) context.Context { - targetDeps.DoneSem() - return nil - }, - }, - Do: func(w *worker.Worker, ctx context.Context) error { + pj := worker2.NewAction(worker2.ActionConfig{ + Name: "pull_meta " + target.Addr, + Deps: []worker2.Dep{pmdeps}, + Ctx: s.sctx, + Hooks: []worker2.Hook{s.tracker.Hook()}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { status.Emit(ctx, tgt.TargetStatus(target, "Scheduling analysis...")) if isSkip { @@ -138,7 +154,7 @@ func (s *schedulerv2) schedule() error { if err != nil { return err } - targetDeps.AddChild(d) + targetDeps.AddDep(d) return nil } @@ -152,12 +168,12 @@ func (s *schedulerv2) schedule() error { if err != nil { return err } - targetDeps.AddChild(g) - + targetDeps.AddDep(g) return nil }, }) - targetDeps.Add(pj) + targetDeps.AddDep(pj) + s.Pool.Schedule(pj) children, err := s.Graph.DAG().GetChildren(target) if err != nil { @@ -165,41 +181,39 @@ func (s *schedulerv2) schedule() error { } for _, child := range children { - s.pullMetaDeps.Get(child.Addr).Add(pj) + s.pullMetaDeps.Get(child.Addr).AddDep(pj) } } - for _, target := range s.toAssess.Slice() { - s.pullMetaDeps.Get(target.Addr).DoneSem() - } - return nil } -func (s *schedulerv2) parentTargetDeps(target specs.Specer) (*worker.WaitGroup, error) { - deps := &worker.WaitGroup{} +func (s *schedulerv2) parentTargetDeps(target specs.Specer) (worker2.Dep, error) { + deps := worker2.NewNamedGroup(xdebug.Sprintf("parent deps: %v", target.Spec().Name)) parents, err := s.Graph.DAG().GetParents(target) if err != nil { return nil, err } for _, parent := range parents { - deps.AddChild(s.deps.Get(parent.Addr)) + deps.AddDep(s.deps.Get(parent.Addr)) } return deps, nil } -func (s *schedulerv2) ScheduleTargetCacheGet(ctx context.Context, target *graph.Target, outputs []string, withRestoreCache, uncompress bool) (*worker.Job, error) { +func (s *schedulerv2) ScheduleTargetCacheGet(ctx context.Context, target *graph.Target, outputs []string, withRestoreCache, uncompress bool) (worker2.Dep, error) { deps, err := s.parentTargetDeps(target) if err != nil { return nil, err } // TODO: add an observability span: OnPullOrGetCache - return s.Pool.Schedule(ctx, &worker.Job{ - Name: "cache get " + target.Addr, - Deps: deps, - Do: func(w *worker.Worker, ctx context.Context) error { + j := worker2.NewAction(worker2.ActionConfig{ + Name: "cache get " + target.Addr, + Ctx: ctx, + Hooks: []worker2.Hook{s.tracker.Hook()}, + Deps: []worker2.Dep{deps}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { cached, err := s.pullOrGetCacheAndPost(ctx, target, outputs, withRestoreCache, false, uncompress) if err != nil { return err @@ -211,10 +225,12 @@ func (s *schedulerv2) ScheduleTargetCacheGet(ctx context.Context, target *graph. return nil }, - }), nil + }) + + return j, nil } -func (s *schedulerv2) ScheduleTargetCacheGetOnce(ctx context.Context, target *graph.Target, outputs []string, withRestoreCache, uncompress bool) (*worker.Job, error) { +func (s *schedulerv2) ScheduleTargetCacheGetOnce(ctx context.Context, target *graph.Target, outputs []string, withRestoreCache, uncompress bool) (worker2.Dep, error) { lock := s.targetSchedLock.Get(target.Addr) lock.Lock() defer lock.Unlock() @@ -228,38 +244,30 @@ func (s *schedulerv2) ScheduleTargetCacheGetOnce(ctx context.Context, target *gr return nil, err } - children, err := s.Graph.DAG().GetChildren(target.Target) - if err != nil { - return nil, err - } - - for _, child := range children { - s.deps.Get(child.Addr).Add(j) - } s.targetSchedJobs.Set(target.Addr, j) return j, nil } -func (s *schedulerv2) ScheduleTargetDepsOnce(ctx context.Context, target specs.Specer) (*worker.WaitGroup, error) { +func (s *schedulerv2) ScheduleTargetDepsOnce(ctx context.Context, target specs.Specer) (worker2.Dep, error) { parents, err := s.Graph.DAG().GetParents(target) if err != nil { return nil, err } - runDeps := &worker.WaitGroup{} + runDeps := worker2.NewNamedGroup(xdebug.Sprintf("schedule target deps once: %v", target.Spec().Name)) for _, parent := range parents { j, err := s.ScheduleTargetGetCacheOrRunOnce(ctx, parent, true, true, true) if err != nil { return nil, err } - runDeps.AddChild(j) + runDeps.AddDep(j) } return runDeps, nil } -func (s *schedulerv2) ScheduleTargetGetCacheOrRunOnce(ctx context.Context, target *graph.Target, allowCached, pullIfCached, uncompress bool) (*worker.WaitGroup, error) { +func (s *schedulerv2) ScheduleTargetGetCacheOrRunOnce(ctx context.Context, target *graph.Target, allowCached, pullIfCached, uncompress bool) (worker2.Dep, error) { l := s.targetSchedLock.Get(target.Addr) l.Lock() defer l.Unlock() @@ -280,11 +288,13 @@ func (s *schedulerv2) ScheduleTargetGetCacheOrRunOnce(ctx context.Context, targe return nil, err } - group := &worker.WaitGroup{} - j := s.Pool.Schedule(ctx, &worker.Job{ - Name: "get cache or run once " + target.Addr, - Deps: deps, - Do: func(w *worker.Worker, ctx context.Context) error { + group := worker2.NewNamedGroup(xdebug.Sprintf("schedule target get cache or run once: %v", target.Spec().Addr)) + j := worker2.NewAction(worker2.ActionConfig{ + Name: "get cache or run once " + target.Addr, + Ctx: ctx, + Deps: []worker2.Dep{deps}, + Hooks: []worker2.Hook{s.tracker.Hook()}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { if target.Cache.Enabled && allowCached { outputs := s.outputs.Get(target.Addr).Slice() @@ -299,7 +309,7 @@ func (s *schedulerv2) ScheduleTargetGetCacheOrRunOnce(ctx context.Context, targe if err != nil { return err } - group.Add(j) + group.AddDep(j) } return nil } @@ -309,19 +319,19 @@ func (s *schedulerv2) ScheduleTargetGetCacheOrRunOnce(ctx context.Context, targe if err != nil { return err } - group.Add(j) + group.AddDep(j) return nil }, }) - group.Add(j) + group.AddDep(j) s.getCacheOrRunSchedJobs.Set(k, group) return group, nil } -func (s *schedulerv2) ScheduleTargetRunOnce(ctx context.Context, target *graph.Target) (*worker.Job, error) { +func (s *schedulerv2) ScheduleTargetRunOnce(ctx context.Context, target *graph.Target) (worker2.Dep, error) { lock := s.targetSchedLock.Get(target.Addr) lock.Lock() defer lock.Unlock() @@ -348,7 +358,7 @@ func (s *schedulerv2) ScheduleTargetRunOnce(ctx context.Context, target *graph.T } for _, child := range children { - s.deps.Get(child.Addr).Add(j) + s.deps.Get(child.Addr).AddDep(j) } s.targetSchedJobs.Set(target.Addr, j) diff --git a/scheduler/target_run.go b/scheduler/target_run.go index 9cceb22c..247d7f02 100644 --- a/scheduler/target_run.go +++ b/scheduler/target_run.go @@ -7,17 +7,17 @@ import ( "github.com/hephbuild/heph/sandbox" "github.com/hephbuild/heph/targetrun" "github.com/hephbuild/heph/utils/locks" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2" ) -func (e *Scheduler) RunWithSpan(ctx context.Context, rr targetrun.Request, iocfg sandbox.IOConfig) (rerr error) { +func (e *Scheduler) RunWithSpan(ctx context.Context, rr targetrun.Request, iocfg sandbox.IOConfig, tracker *worker2.RunningTracker) (rerr error) { ctx, rspan := e.Observability.SpanRun(ctx, rr.Target) defer rspan.EndError(rerr) - return e.Run(ctx, rr, iocfg) + return e.Run(ctx, rr, iocfg, tracker) } -func (e *Scheduler) Run(ctx context.Context, rr targetrun.Request, iocfg sandbox.IOConfig) error { +func (e *Scheduler) Run(ctx context.Context, rr targetrun.Request, iocfg sandbox.IOConfig, tracker *worker2.RunningTracker) error { if err := ctx.Err(); err != nil { return err } @@ -55,7 +55,7 @@ func (e *Scheduler) Run(ctx context.Context, rr targetrun.Request, iocfg sandbox return fmt.Errorf("args are not supported with cache") } - cached, err := e.pullOrGetCacheAndPost(ctx, target, target.OutWithSupport.Names(), false, true, false) + cached, err := e.pullOrGetCacheAndPost(ctx, target, target.Out.Names(), false, true, false) if err != nil { return err } @@ -74,7 +74,7 @@ func (e *Scheduler) Run(ctx context.Context, rr targetrun.Request, iocfg sandbox rr.Compress = len(writeableCaches) > 0 } - rtarget, err := e.Runner.Run(ctx, rr, iocfg) + rtarget, err := e.Runner.Run(ctx, rr, iocfg, tracker) if err != nil { return targetrun.WrapTargetFailed(err, target) } @@ -86,11 +86,7 @@ func (e *Scheduler) Run(ctx context.Context, rr targetrun.Request, iocfg sandbox if len(writeableCaches) > 0 { for _, cache := range writeableCaches { - j := e.scheduleStoreExternalCache(ctx, rtarget.Target, cache) - - if poolDeps := poolwait.ForegroundWaitGroup(ctx); poolDeps != nil { - poolDeps.Add(j) - } + _ = e.scheduleStoreExternalCache(ctx, rtarget.Target, cache, []*worker2.RunningTracker{tracker, e.BackgroundTracker}) } } diff --git a/specs/target_spec.go b/specs/target_spec.go index c6428520..fe5fde11 100644 --- a/specs/target_spec.go +++ b/specs/target_spec.go @@ -135,6 +135,7 @@ type Target struct { Timeout time.Duration GenDepsMeta bool Annotations map[string]interface{} + Requests map[string]float64 } func (t Target) MarshalJSON() ([]byte, error) { diff --git a/specs/target_spec_equal.go b/specs/target_spec_equal.go index bdf24c75..3ab1e704 100644 --- a/specs/target_spec_equal.go +++ b/specs/target_spec_equal.go @@ -179,6 +179,14 @@ func (t Target) equalStruct(spec Target) bool { return false } + if !mapEqual(t.Annotations, t.Annotations) { + return false + } + + if !mapEqual(t.Requests, t.Requests) { + return false + } + return true } diff --git a/targetrun/prepare.go b/targetrun/prepare.go index 72e09493..10cb3dd1 100644 --- a/targetrun/prepare.go +++ b/targetrun/prepare.go @@ -26,7 +26,7 @@ import ( "github.com/hephbuild/heph/utils/locks" "github.com/hephbuild/heph/utils/xfs" "github.com/hephbuild/heph/utils/xmath" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "io" "os" "path/filepath" @@ -44,7 +44,7 @@ type Runner struct { QueryFunctions func(*graph.Target) map[string]exprs.Func Cwd string Config *config.Config - Pool *worker.Pool + Pool *worker2.Engine } type Target struct { @@ -285,7 +285,10 @@ func (e *Runner) runPrepare(ctx context.Context, target *graph.Target, rr Reques linkSrcRec.Add("", file.Abs(), file.RelRoot(), "") } } else { - art := dept.Artifacts.OutTar(dep.Output) + art, ok := dept.Artifacts.OutTar2(dep.Output) + if !ok { + return nil, fmt.Errorf("artifact %v|%v not found: %v", dept.Addr, dep.Output, dept.ActualOutFiles().Names()) + } p, stats, err := e.LocalCache.UncompressedPathFromArtifact(ctx, dept, art) if err != nil { return nil, err @@ -411,6 +414,7 @@ func (e *Runner) runPrepare(ctx context.Context, target *graph.Target, rr Reques // Forward heph variables inside the sandbox forward := []string{ "HEPH_PROFILES", + "HEPH_DEBUG_POOLWAIT", "HEPH_FROM_PATH", "HEPH_CLOUD_TOKEN", hephprovider.EnvSrcRoot, diff --git a/targetrun/run.go b/targetrun/run.go index 51fb9618..fadc611c 100644 --- a/targetrun/run.go +++ b/targetrun/run.go @@ -15,8 +15,7 @@ import ( "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/tgt" "github.com/hephbuild/heph/utils/xfs" - "github.com/hephbuild/heph/worker" - "github.com/hephbuild/heph/worker/poolwait" + "github.com/hephbuild/heph/worker2" "os" "path/filepath" "strconv" @@ -41,7 +40,7 @@ type RequestOpts struct { PullCache bool } -func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig) (*Target, error) { +func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig, tracker *worker2.RunningTracker) (*Target, error) { target := rr.Target rtarget, err := e.runPrepare(ctx, rr.Target, rr) @@ -144,7 +143,7 @@ func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig) (* sandbox.AddPathEnv(env, binDir, target.Sandbox && !hasPathInEnv) execCtx := ctx - if target.Timeout > 0 { + if !rr.Shell && target.Timeout > 0 { var cancel context.CancelFunc execCtx, cancel = context.WithTimeout(ctx, target.Timeout) defer cancel() @@ -235,7 +234,7 @@ func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig) (* } if cerr := ctx.Err(); cerr != nil { - if !errors.Is(cerr, err) { + if !errors.Is(err, cerr) { err = fmt.Errorf("%w: %v", cerr, err) } } @@ -281,11 +280,13 @@ func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig) (* } if !e.Config.Engine.KeepSandbox { - j := e.Pool.Schedule(ctx, &worker.Job{ - Name: fmt.Sprintf("clear sandbox %v", target.Addr), + e.Pool.Schedule(worker2.NewAction(worker2.ActionConfig{ + Name: fmt.Sprintf("clear sandbox %v", target.Addr), + Ctx: ctx, + Hooks: []worker2.Hook{tracker.Hook()}, // We need to make sure to wait for the lock to be released before proceeding - Deps: worker.WaitGroupChan(completedCh), - Do: func(w *worker.Worker, ctx context.Context) error { + Deps: []worker2.Dep{worker2.NewChanDep(ctx, completedCh)}, + Do: func(ctx context.Context, ins worker2.InStore, outs worker2.OutStore) error { locked, err := rtarget.SandboxLock.TryLock(ctx) if err != nil { return err @@ -310,10 +311,7 @@ func (e *Runner) Run(ctx context.Context, rr Request, iocfg sandbox.IOConfig) (* return nil }, - }) - if poolDeps := poolwait.ForegroundWaitGroup(ctx); poolDeps != nil { - poolDeps.Add(j) - } + })) } return rtarget, nil diff --git a/test/e2e/lib/cache.go b/test/e2e/lib/cache.go index 7da0d244..cea60ca9 100644 --- a/test/e2e/lib/cache.go +++ b/test/e2e/lib/cache.go @@ -56,7 +56,7 @@ func ValidateCache(tgt string, outputs []string, fromRemote, compressed, uncompr } if len(outputs) > 0 { expected = append(expected, "_output") - expected = append(expected, "_output_hash") + expected = append(expected, "_output_meta") } for _, output := range outputs { expected = append(expected, "hash_out_"+output) diff --git a/test/go/e2e.BUILD b/test/go/e2e.BUILD index 2c2bde04..ab09aa50 100644 --- a/test/go/e2e.BUILD +++ b/test/go/e2e.BUILD @@ -3,7 +3,7 @@ load("//test", "e2e_test") e2e_test( name = "sanity_go_version", cmd = "heph run //test/go:version", - expect_output_contains = "go version go1.21.4", + expect_output_contains = "go version go1.22.2", ) e2e_test( diff --git a/utils/ads/ads.go b/utils/ads/ads.go index 3d86d10d..afeb7fc9 100644 --- a/utils/ads/ads.go +++ b/utils/ads/ads.go @@ -57,6 +57,16 @@ func ContainsAny[T comparable](a []T, e []T) bool { return false } +func ContainsAll[T comparable](a []T, e []T) bool { + for _, ee := range e { + if !Contains(a, ee) { + return false + } + } + + return true +} + func Filter[S ~[]E, E any](a S, f func(E) bool) S { o := a alloc := false diff --git a/utils/ads/dedup.go b/utils/ads/dedup.go index 253c382c..11952410 100644 --- a/utils/ads/dedup.go +++ b/utils/ads/dedup.go @@ -20,6 +20,12 @@ func DedupAppend[T any, K comparable](as []T, id func(T) K, vs ...T) []T { return as } +func DedupAppenderIdentity[T comparable](as []T, cap int) func([]T, T) []T { + return DedupAppender(as, func(t T) T { + return t + }, cap) +} + func DedupAppender[T any, K comparable](as []T, id func(T) K, cap int) func([]T, T) []T { value := make(map[K]struct{}, len(as)+cap) for _, a := range as { diff --git a/utils/flock/flock.go b/utils/flock/flock.go index d1002744..3b741bec 100644 --- a/utils/flock/flock.go +++ b/utils/flock/flock.go @@ -1,7 +1,9 @@ package flock import ( + "errors" "fmt" + "golang.org/x/sys/unix" "os" "syscall" ) @@ -58,3 +60,12 @@ func Flunlock(f *os.File) error { return nil } + +func IsErrWouldBlock(err error) bool { + var errno unix.Errno + if ok := errors.As(err, &errno); ok && errno == unix.EWOULDBLOCK { + return true + } + + return false +} diff --git a/utils/hash/hash.go b/utils/hash/hash.go index ebd64005..552bca5f 100644 --- a/utils/hash/hash.go +++ b/utils/hash/hash.go @@ -5,7 +5,6 @@ import ( "encoding/hex" "github.com/hephbuild/heph/utils/xsync" "github.com/zeebo/xxh3" - "golang.org/x/exp/slices" "io" ) @@ -20,32 +19,6 @@ type Hash interface { io.Writer } -func HashArray[T any](h Hash, a []T, f func(T) string) { - entries := make([]string, 0, len(a)) - for _, e := range a { - entries = append(entries, f(e)) - } - - slices.Sort(entries) - - for _, entry := range entries { - h.String(entry) - } -} - -func HashMap[K comparable, V any](h Hash, a map[K]V, f func(K, V) string) { - entries := make([]string, 0, len(a)) - for k, v := range a { - entries = append(entries, f(k, v)) - } - - slices.Sort(entries) - - for _, entry := range entries { - h.String(entry) - } -} - func NewHash() Hash { h := &hasher{ h: xxh3.New(), @@ -58,7 +31,7 @@ type hasher struct { h *xxh3.Hasher } -func (h *hasher) Write(p []byte) (n int, err error) { +func (h *hasher) Write(p []byte) (int, error) { return h.h.Write(p) } @@ -101,15 +74,3 @@ func (h *hasher) Sum() string { hb := h.h.Sum128().Bytes() return hex.EncodeToString(hb[:]) } - -func HashString(s string) string { - h := NewHash() - h.String(s) - return h.Sum() -} - -func HashBytes(s []byte) string { - h := NewHash() - h.Write(s) - return h.Sum() -} diff --git a/utils/hash/utils.go b/utils/hash/utils.go new file mode 100644 index 00000000..900fccd3 --- /dev/null +++ b/utils/hash/utils.go @@ -0,0 +1,41 @@ +package hash + +import "golang.org/x/exp/slices" + +func HashArray[T any](h Hash, a []T, f func(T) string) { + entries := make([]string, 0, len(a)) + for _, e := range a { + entries = append(entries, f(e)) + } + + slices.Sort(entries) + + for _, entry := range entries { + h.String(entry) + } +} + +func HashMap[K comparable, V any](h Hash, a map[K]V, f func(K, V) string) { + entries := make([]string, 0, len(a)) + for k, v := range a { + entries = append(entries, f(k, v)) + } + + slices.Sort(entries) + + for _, entry := range entries { + h.String(entry) + } +} + +func HashString(s string) string { + h := NewHash() + h.String(s) + return h.Sum() +} + +func HashBytes(s []byte) string { + h := NewHash() + h.Write(s) + return h.Sum() +} diff --git a/utils/locks/flock.go b/utils/locks/flock.go index 7f1f631a..1ab31248 100644 --- a/utils/locks/flock.go +++ b/utils/locks/flock.go @@ -2,14 +2,12 @@ package locks import ( "context" - "errors" "fmt" log "github.com/hephbuild/heph/log/liblog" "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/utils/flock" "github.com/hephbuild/heph/utils/xfs" - "github.com/hephbuild/heph/worker" - "golang.org/x/sys/unix" + "github.com/hephbuild/heph/worker2" "os" "strconv" "sync" @@ -17,7 +15,7 @@ import ( "time" ) -func NewFlock(name, p string) RWLocker { +func NewFlock(name, p string) *Flock { if name == "" || log.Default().IsLevelEnabled(log.DebugLevel) { name = p } @@ -63,8 +61,7 @@ func (l *Flock) tryLock(ctx context.Context, ro bool, onErr func(f *os.File, ro logger.Debugf("Attempting to acquire lock for %s...", f.Name()) err = flock.Flock(f, ro, false) if err != nil { - var errno unix.Errno - if ok := errors.As(err, &errno); ok && errno == unix.EWOULDBLOCK { + if flock.IsErrWouldBlock(err) { ok, err := onErr(f, ro) if err != nil { return false, fmt.Errorf("acquire lock for %s: %w", l.name, err) @@ -153,14 +150,7 @@ func (l *Flock) lock(ctx context.Context, ro bool) error { lockCh <- flock.Flock(f, ro, true) }() - err := worker.SuspendE(ctx, func() error { - select { - case err := <-lockCh: - return err - case <-ctx.Done(): - return ctx.Err() - } - }) + err := worker2.WaitChanE(ctx, lockCh) if err != nil { return false, err } diff --git a/utils/sets/set.go b/utils/sets/set.go index 29d6be9d..0a7a5e9e 100644 --- a/utils/sets/set.go +++ b/utils/sets/set.go @@ -1,7 +1,7 @@ package sets import ( - "github.com/hephbuild/heph/utils/ads" + "slices" "sync" ) @@ -71,14 +71,18 @@ func (ts *Set[K, T]) GetKey(k K) T { func (ts *Set[K, T]) has(t T) bool { k := ts.f(t) - _, ok := ts.m[k] + return ts.hask(k) +} +func (ts *Set[K, T]) hask(k K) bool { + _, ok := ts.m[k] return ok } func (ts *Set[K, T]) add(t T) bool { k := ts.f(t) - if ts.has(t) { + + if ts.hask(k) { return false } @@ -125,14 +129,21 @@ func (ts *Set[K, T]) Copy() *Set[K, T] { } func (ts *Set[K, T]) Pop(i int) T { + v := ts.a[i] + + ts.Remove(v) + + return v +} + +func (ts *Set[K, T]) Remove(v T) { ts.mu.Lock() defer ts.mu.Unlock() - v := ts.a[i] + k := ts.f(v) - ts.a = ads.Filter(ts.a, func(t T) bool { - return ts.f(t) != ts.f(v) + delete(ts.m, k) + ts.a = slices.DeleteFunc(ts.a, func(t T) bool { + return ts.f(t) == k }) - - return v } diff --git a/utils/xcontext/context.go b/utils/xcontext/context.go index d895ef2f..ffde2b94 100644 --- a/utils/xcontext/context.go +++ b/utils/xcontext/context.go @@ -5,6 +5,7 @@ import ( "github.com/hephbuild/heph/log/log" "github.com/hephbuild/heph/utils/ads" "github.com/hephbuild/heph/utils/xsync" + "github.com/hephbuild/heph/utils/xtea" "os" "os/signal" "sync" @@ -138,10 +139,12 @@ func Cancel(ctx context.Context) { cancel() } +const stuckTimeout = 5 * time.Second + func BootstrapSoftCancel() (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) - sigCh := make(chan os.Signal, 1) + sigCh := make(chan os.Signal) signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM) sc := newSoftCancelState() @@ -160,21 +163,20 @@ func BootstrapSoftCancel() (context.Context, context.CancelFunc) { }() select { case <-sigCh: - case <-time.After(5 * time.Second): } log.Warnf("Forcing cancellation...") hardCanceled = true sc.hardCancel() select { - // Wait for soft cancel to all be unregistered, should be instant, unless something is stuck + // Wait for soft cancel to all be unregistered, should be fast, unless something is stuck case <-sc.wait(): // Wait for graceful exit - <-time.After(2 * time.Second) - case <-time.After(2 * time.Second): + <-time.After(stuckTimeout) + case <-time.After(stuckTimeout): // All soft cancel did not unregister, something is stuck... } } else { - <-time.After(2 * time.Second) + <-time.After(stuckTimeout) } log.Error("Something seems to be stuck, ctrl+c one more time to forcefully exit") @@ -183,6 +185,7 @@ func BootstrapSoftCancel() (context.Context, context.CancelFunc) { if sig, ok := sig.(syscall.Signal); ok { sigN = int(sig) } + xtea.ResetTerminal() os.Exit(128 + sigN) }() diff --git a/utils/xdebug/fmt.go b/utils/xdebug/fmt.go new file mode 100644 index 00000000..7bc88814 --- /dev/null +++ b/utils/xdebug/fmt.go @@ -0,0 +1,13 @@ +package xdebug + +import "fmt" + +const enabled = true + +// Sprintf is fmt.Sprintf, but zero-allocation/overhead when debug is disabled +func Sprintf(format string, a ...any) string { + if enabled { + return fmt.Sprintf(format, a...) + } + return format +} diff --git a/utils/xio/copy.go b/utils/xio/copy.go index a63c2e6b..90a2a8a7 100644 --- a/utils/xio/copy.go +++ b/utils/xio/copy.go @@ -18,15 +18,13 @@ func CopyBuffer(dst io.Writer, src io.Reader, buf []byte, f func(written int64)) w := io.MultiWriter(dst, t) written, err := io.CopyBuffer(w, src, buf) + if err != nil { + return written, err + } if written == 0 { - _, err := w.Write([]byte{}) - if err != nil { - return written, err - } + _, _ = w.Write([]byte{}) } - f(written) - - return written, err + return written, nil } diff --git a/utils/xio/tracker.go b/utils/xio/tracker.go index b2ab5f72..e31cdd10 100644 --- a/utils/xio/tracker.go +++ b/utils/xio/tracker.go @@ -3,7 +3,6 @@ package xio type Tracker struct { Written int64 OnWrite func(written int64) - OnRead func(written int64) } func (t *Tracker) Write(b []byte) (int, error) { diff --git a/utils/xstarlark/fmt/fmt.go b/utils/xstarlark/fmt/fmt.go index 08b64276..71aad2aa 100644 --- a/utils/xstarlark/fmt/fmt.go +++ b/utils/xstarlark/fmt/fmt.go @@ -533,7 +533,7 @@ func (f *formatter) formatExpr(w Writer, expr syntax.Expr) error { case syntax.INT: w.WriteString(strconv.FormatInt(expr.Value.(int64), 10)) case syntax.FLOAT: - w.WriteString(strconv.FormatFloat(expr.Value.(float64), 'f', 0, 64)) + w.WriteString(strconv.FormatFloat(expr.Value.(float64), 'f', -1, 64)) default: return fmt.Errorf("unhandled Literal token: %v", expr.Token) } diff --git a/utils/xstarlark/fmt/testdata/float.txt b/utils/xstarlark/fmt/testdata/float.txt new file mode 100644 index 00000000..a98eeb93 --- /dev/null +++ b/utils/xstarlark/fmt/testdata/float.txt @@ -0,0 +1,3 @@ +1.5 +---- +1.5 diff --git a/utils/xtea/program.go b/utils/xtea/program.go index 500a7c42..114c3982 100644 --- a/utils/xtea/program.go +++ b/utils/xtea/program.go @@ -3,7 +3,6 @@ package xtea import ( tea "github.com/charmbracelet/bubbletea" "github.com/hephbuild/heph/log/log" - "github.com/hephbuild/heph/utils/xpanic" ) func RunModel(model tea.Model, opts ...tea.ProgramOption) error { @@ -14,12 +13,11 @@ func RunModel(model tea.Model, opts ...tea.ProgramOption) error { func Run(p *tea.Program) error { defer func() { + SetResetTerminal(nil) log.SetDiversion(nil) - _ = xpanic.Recover(func() error { - return p.ReleaseTerminal() - }) }() + SetResetTerminal(p) _, err := p.Run() return err } diff --git a/utils/xtea/singletui.go b/utils/xtea/singletui.go index 047844a0..ebedd059 100644 --- a/utils/xtea/singletui.go +++ b/utils/xtea/singletui.go @@ -2,6 +2,7 @@ package xtea import ( "fmt" + tea "github.com/charmbracelet/bubbletea" "runtime/debug" "sync" ) @@ -28,6 +29,27 @@ func SingleflightTry() bool { } func SingleflightDone() { - //tuiStack = nil + if debugSingleflight { + tuiStack = nil + } tuim.Unlock() } + +var resetTermFunc func() + +func SetResetTerminal(p *tea.Program) { + if p == nil { + resetTermFunc = nil + return + } + + resetTermFunc = func() { + _ = p.ReleaseTerminal() + } +} + +func ResetTerminal() { + if f := resetTermFunc; f != nil { + f() + } +} diff --git a/utils/xtypes/xtypes.go b/utils/xtypes/xtypes.go index 17f96614..d8b3548e 100644 --- a/utils/xtypes/xtypes.go +++ b/utils/xtypes/xtypes.go @@ -1,8 +1,25 @@ package xtypes +import "reflect" + // ForceCast force casting any type by going through any // It is pretty somewhat unsafe, beware! func ForceCast[B any](a any) B { var b = a.(B) return b } + +// IsNil allows to go around nil interfaces +// see https://stackoverflow.com/a/78104852/3212099 +func IsNil(input interface{}) bool { + if input == nil { + return true + } + kind := reflect.ValueOf(input).Kind() + switch kind { + case reflect.Ptr, reflect.Map, reflect.Slice, reflect.Chan: + return reflect.ValueOf(input).IsNil() + default: + return false + } +} diff --git a/worker/error.go b/worker/error.go deleted file mode 100644 index d99c91a4..00000000 --- a/worker/error.go +++ /dev/null @@ -1,89 +0,0 @@ -package worker - -import ( - "errors" - "fmt" - "go.uber.org/multierr" -) - -type JobError struct { - ID uint64 - Name string - State JobState - Err error - - root error -} - -func (e JobError) Skipped() bool { - return e.State == StateSkipped -} - -func CollectUniqueErrors(inErrs []error) []error { - var errs []error - jerrs := map[uint64]JobError{} - - for _, err := range inErrs { - var jerr JobError - if errors.As(err, &jerr) { - jerrs[jerr.ID] = jerr - } else { - errs = append(errs, err) - } - } - - for _, err := range jerrs { - errs = append(errs, err) - } - - return errs -} - -func CollectRootErrors(err error) error { - errs := make([]error, 0) - - for _, err := range multierr.Errors(err) { - var jerr JobError - if errors.As(err, &jerr) { - errs = append(errs, jerr.Root()) - } else { - errs = append(errs, err) - } - } - - return multierr.Combine(CollectUniqueErrors(errs)...) -} - -func (e JobError) Root() error { - if e.root != nil { - return e.root - } - - var err error = e - for { - var jerr JobError - if errors.As(err, &jerr) { - if jerr.Skipped() { - if _, ok := jerr.Err.(JobError); !ok { - break - } - - err = jerr.Err - continue - } - } - - break - } - - e.root = err - return err -} - -func (e JobError) Unwrap() error { - return e.Err -} - -func (e JobError) Error() string { - return fmt.Sprintf("%v is %v: %v", e.Name, e.State, e.Err) -} diff --git a/worker/poolwait/foregroundwg.go b/worker/poolwait/foregroundwg.go deleted file mode 100644 index 73af0d73..00000000 --- a/worker/poolwait/foregroundwg.go +++ /dev/null @@ -1,44 +0,0 @@ -package poolwait - -import ( - "context" - "github.com/hephbuild/heph/worker" -) - -type keyFgWaitGroup struct{} - -func ContextWithForegroundWaitGroup(ctx context.Context) (context.Context, *worker.WaitGroup) { - deps := &worker.WaitGroup{} - ctx = context.WithValue(ctx, keyFgWaitGroup{}, deps) - - return ctx, deps -} - -// GCWaitGroup allows to depend on other Job/Worker without keeping them in memory, allowing GC to work -type GCWaitGroup struct { - Deps *worker.WaitGroup -} - -func (wwg *GCWaitGroup) Add(j *worker.Job) { - wwg.Deps.Add(j) - go func() { - <-j.Wait() - wwg.Deps.Remove(j) - }() -} - -func (wwg *GCWaitGroup) AddChild(wg *worker.WaitGroup) { - wwg.Deps.AddChild(wg) - go func() { - <-wg.Done() - wwg.Deps.RemoveChild(wg) - }() -} - -func ForegroundWaitGroup(ctx context.Context) *GCWaitGroup { - if deps, ok := ctx.Value(keyFgWaitGroup{}).(*worker.WaitGroup); ok { - return &GCWaitGroup{Deps: deps} - } - - return nil -} diff --git a/worker/poolwait/tui.go b/worker/poolwait/tui.go deleted file mode 100644 index ec98746a..00000000 --- a/worker/poolwait/tui.go +++ /dev/null @@ -1,31 +0,0 @@ -package poolwait - -import ( - "context" - "fmt" - "github.com/hephbuild/heph/utils/xtea" - "github.com/hephbuild/heph/worker" - "github.com/hephbuild/heph/worker/poolui" -) - -func termUI(ctx context.Context, name string, deps *worker.WaitGroup, pool *worker.Pool) error { - if !xtea.SingleflightTry() { - return logUI(name, deps, pool) - } - - defer xtea.SingleflightDone() - - m := poolui.New(ctx, name, deps, pool, true) - defer m.Clean() - - err := xtea.RunModel(m) - if err != nil { - return err - } - - if !deps.IsDone() { - pool.Stop(fmt.Errorf("TUI exited unexpectedly")) - } - - return nil -} diff --git a/worker/poolwait/wait.go b/worker/poolwait/wait.go deleted file mode 100644 index ed86f076..00000000 --- a/worker/poolwait/wait.go +++ /dev/null @@ -1,46 +0,0 @@ -package poolwait - -import ( - "context" - "errors" - "fmt" - "github.com/hephbuild/heph/log/log" - "github.com/hephbuild/heph/utils/xtea" - "github.com/hephbuild/heph/worker" - "go.uber.org/multierr" -) - -func Wait(ctx context.Context, name string, pool *worker.Pool, deps *worker.WaitGroup, plain bool) error { - useTUI := xtea.IsTerm() && !plain - - log.Tracef("WaitPool %v", name) - defer func() { - log.Tracef("WaitPool %v DONE", name) - }() - - if useTUI { - err := termUI(ctx, name, deps, pool) - if err != nil { - return fmt.Errorf("poolui: %w", err) - } - } else { - err := logUI(name, deps, pool) - if err != nil { - return fmt.Errorf("logpoolui: %w", err) - } - } - - perr := pool.Err() - derr := deps.Err() - - if perr != nil && derr != nil { - if errors.Is(perr, derr) || errors.Is(derr, perr) || derr == perr { - return perr - } - - perr = fmt.Errorf("pool: %w", perr) - derr = fmt.Errorf("deps: %w", derr) - } - - return multierr.Combine(perr, derr) -} diff --git a/worker/utils.go b/worker/utils.go deleted file mode 100644 index 628817c0..00000000 --- a/worker/utils.go +++ /dev/null @@ -1,104 +0,0 @@ -package worker - -import ( - "context" -) - -func WaitGroupOr(wgs ...*WaitGroup) *WaitGroup { - switch len(wgs) { - case 0: - panic("at least one WaitGroup required") - case 1: - return wgs[0] - } - - doneCh := make(chan struct{}) - - for _, wg := range wgs { - wg := wg - - go func() { - select { - case <-doneCh: - case <-wg.Done(): - close(doneCh) - } - }() - } - - return WaitGroupChan(doneCh) -} - -func WaitGroupJob(j *Job) *WaitGroup { - wg := &WaitGroup{} - wg.Add(j) - - return wg -} - -func WaitGroupChan[T any](ch <-chan T) *WaitGroup { - wg := &WaitGroup{} - wg.AddSem() - go func() { - <-ch - wg.DoneSem() - }() - - return wg -} - -type poolKey struct{} -type jobKey struct{} - -func ContextWithPoolJob(ctx context.Context, p *Pool, j *Job) context.Context { - ctx = context.WithValue(ctx, poolKey{}, p) - ctx = context.WithValue(ctx, jobKey{}, j) - - return ctx -} - -func PoolJobFromContext(ctx context.Context) (*Pool, *Job, bool) { - p, _ := ctx.Value(poolKey{}).(*Pool) - j, _ := ctx.Value(jobKey{}).(*Job) - - return p, j, p != nil -} - -func Suspend(ctx context.Context, f func()) { - p, j, ok := PoolJobFromContext(ctx) - - // We are not running in a worker, we can safely wait normally - if !ok { - f() - return - } - - // Detach from worker - resumeCh := j.suspend() - - // Wait for it to be suspended - <-j.suspendedCh - - // Wait for condition - f() - - // Reassign worker, which will resume execution by closing resumeCh - p.jobsCh <- j - - <-resumeCh -} - -func SuspendE(ctx context.Context, f func() error) error { - var err error - Suspend(ctx, func() { - err = f() - }) - return err -} - -func SuspendWaitGroup(ctx context.Context, wg *WaitGroup) error { - return SuspendE(ctx, func() error { - <-wg.Done() - return wg.Err() - }) -} diff --git a/worker/waitgroup.go b/worker/waitgroup.go deleted file mode 100644 index e0ac2603..00000000 --- a/worker/waitgroup.go +++ /dev/null @@ -1,280 +0,0 @@ -package worker - -import ( - "errors" - "fmt" - "github.com/hephbuild/heph/utils/ads" - "go.uber.org/multierr" - "sync" - "sync/atomic" -) - -type WaitGroup struct { - m sync.RWMutex - wgs []*WaitGroup - jobs []*Job - doneCh chan struct{} - err error - cond *sync.Cond - oSetup sync.Once - sem int64 - - failfast bool -} - -func (wg *WaitGroup) Add(job *Job) { - if job == nil { - panic("job cannot be nil") - } - - wg.m.Lock() - defer wg.m.Unlock() - - go func() { - <-job.Wait() - wg.handleUnitDone() - }() - - wg.jobs = append(wg.jobs, job) -} - -func (wg *WaitGroup) Remove(job *Job) { - wg.m.Lock() - defer wg.m.Unlock() - - wg.jobs = ads.Filter(wg.jobs, func(wgjob *Job) bool { - return job != wgjob - }) -} - -func (wg *WaitGroup) AddChild(child *WaitGroup) { - if child == nil { - panic("child cannot be nil") - } - - wg.m.Lock() - defer wg.m.Unlock() - - go func() { - <-child.Done() - wg.handleUnitDone() - }() - - wg.wgs = append(wg.wgs, child) -} - -func (wg *WaitGroup) RemoveChild(child *WaitGroup) { - wg.m.Lock() - defer wg.m.Unlock() - - wg.wgs = ads.Filter(wg.wgs, func(wgchild *WaitGroup) bool { - return child != wgchild - }) -} - -func (wg *WaitGroup) handleUnitDone() { - wg.broadcast() -} - -func (wg *WaitGroup) AddSem() { - atomic.AddInt64(&wg.sem, 1) -} - -func (wg *WaitGroup) DoneSem() { - v := atomic.AddInt64(&wg.sem, -1) - if v < 0 { - panic("too many calls to DoneSem") - } else if v == 0 { - wg.handleUnitDone() - } -} - -func (wg *WaitGroup) Jobs() []*Job { - jobs := wg.jobs[:] - for _, wg := range wg.wgs[:] { - jobs = append(jobs, wg.Jobs()...) - } - - return jobs -} - -func (wg *WaitGroup) broadcast() { - if wg.cond == nil { - return - } - - wg.cond.L.Lock() - defer wg.cond.L.Unlock() - - wg.cond.Broadcast() -} - -func (wg *WaitGroup) wait() { - wg.cond.L.Lock() - defer wg.cond.L.Unlock() - - var err error - for { - err = wg.keepWaiting(wg.failfast) - if !errors.Is(err, ErrPending) { - break - } - - wg.cond.Wait() - } - - if wg.err == nil { - wg.err = err - } - close(wg.doneCh) -} - -func (wg *WaitGroup) Done() <-chan struct{} { - wg.oSetup.Do(func() { - wg.cond = sync.NewCond(&wg.m) - wg.doneCh = make(chan struct{}) - go wg.wait() - }) - - return wg.doneCh -} - -func (wg *WaitGroup) IsDone() bool { - select { - case <-wg.Done(): - return true - default: - return false - } -} - -func (wg *WaitGroup) Err() error { - return wg.err -} - -func (wg *WaitGroup) walkerTransitiveDo(mj map[uint64]struct{}, mwg map[*WaitGroup]struct{}, f func(j *Job)) { - if _, ok := mwg[wg]; ok { - return - } - mwg[wg] = struct{}{} - - for _, job := range wg.jobs { - if _, ok := mj[job.ID]; ok { - continue - } - mj[job.ID] = struct{}{} - - f(job) - - job.Deps.walkerTransitiveDo(mj, mwg, f) - } - - for _, wg := range wg.wgs { - wg.walkerTransitiveDo(mj, mwg, f) - } -} - -func (wg *WaitGroup) TransitiveDo(f func(j *Job)) { - mj := map[uint64]struct{}{} - mwg := map[*WaitGroup]struct{}{} - wg.walkerTransitiveDo(mj, mwg, f) -} - -type WaitGroupStats struct { - All uint64 - Done uint64 - Success uint64 - Failed uint64 - Skipped uint64 - Suspended uint64 -} - -func (wg *WaitGroup) TransitiveCount() WaitGroupStats { - s := WaitGroupStats{} - - wg.TransitiveDo(func(j *Job) { - atomic.AddUint64(&s.All, 1) - - if j.IsDone() { - atomic.AddUint64(&s.Done, 1) - } - - switch j.State { - case StateSuccess: - atomic.AddUint64(&s.Success, 1) - case StateFailed: - atomic.AddUint64(&s.Failed, 1) - case StateSkipped: - atomic.AddUint64(&s.Skipped, 1) - case StateSuspended: - atomic.AddUint64(&s.Suspended, 1) - } - }) - - return s -} - -var ErrPending = errors.New("pending") - -var ErrSemPending = fmt.Errorf("sem is > 0: %w", ErrPending) - -// keepWaiting returns ErrPending if it should keep waiting, nil represents no jobs error -func (wg *WaitGroup) keepWaiting(failFast bool) error { - if atomic.LoadInt64(&wg.sem) > 0 { - return ErrSemPending - } - - var errs []error - jerrs := map[uint64]JobError{} - addErr := func(err error) { - var jerr JobError - if errors.As(err, &jerr) { - jerrs[jerr.ID] = jerr - } else { - errs = append(errs, err) - } - } - - for _, wg := range wg.wgs { - select { - case <-wg.Done(): - err := wg.Err() - if err != nil { - if failFast { - return err - } else { - for _, err := range multierr.Errors(err) { - addErr(err) - } - } - } - - default: - return ErrPending - } - } - - for _, job := range wg.jobs { - state := job.State - - if state.IsDone() { - if state == StateSuccess { - continue - } - - if failFast { - return job.err - } else { - addErr(job.err) - } - } else { - return ErrPending - } - } - - for _, err := range jerrs { - errs = append(errs, err) - } - - return multierr.Combine(errs...) -} diff --git a/worker/waitgroup_test.go b/worker/waitgroup_test.go deleted file mode 100644 index b0d75e5e..00000000 --- a/worker/waitgroup_test.go +++ /dev/null @@ -1,218 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/multierr" - "sync/atomic" - "testing" - "time" -) - -type tracker struct { - c int32 -} - -func job(t *testing.T, tr *tracker, id string, d time.Duration) *Job { - t.Helper() - - return &Job{ - Name: id, - Do: func(w *Worker, ctx context.Context) error { - //t.Log(id) - //defer t.Log(id, "done") - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(d): - if tr != nil { - atomic.AddInt32(&tr.c, 1) - } - return nil - } - }, - } -} - -func TestSanity(t *testing.T) { - t.Parallel() - - p := NewPool(1) - defer p.Stop(nil) - ctx := context.Background() - - tr := &tracker{} - - j := p.Schedule(ctx, job(t, tr, "j1", time.Second)) - - deps := &WaitGroup{} - deps.Add(j) - - <-deps.Done() - require.NoError(t, deps.Err()) - - assert.Equal(t, int32(1), tr.c) -} - -func TestErrorFailFast(t *testing.T) { - t.Parallel() - - p := NewPool(4) - defer p.Stop(nil) - ctx := context.Background() - - tr := &tracker{} - deps := &WaitGroup{ - failfast: true, - } - - jerr := p.Schedule(ctx, &Job{ - Name: "jerr", - Do: func(w *Worker, ctx context.Context) error { - return fmt.Errorf("failing") - }, - }) - deps.Add(jerr) - - for i := 0; i < 3; i++ { - j := p.Schedule(ctx, job(t, tr, fmt.Sprintf("j%v", i), time.Second)) - deps.Add(j) - } - - <-deps.Done() - assert.ErrorContains(t, deps.Err(), "failing") - - assert.Equal(t, int32(0), tr.c) -} - -func TestErrorFailSlow(t *testing.T) { - t.Parallel() - - p := NewPool(4) - defer p.Stop(nil) - ctx := context.Background() - - tr := &tracker{} - deps := &WaitGroup{} - - jerr := p.Schedule(ctx, &Job{ - Name: "jerr", - Do: func(w *Worker, ctx context.Context) error { - return fmt.Errorf("failing") - }, - }) - deps.Add(jerr) - - for i := 0; i < 3; i++ { - j := p.Schedule(ctx, job(t, tr, fmt.Sprintf("j%v", i), time.Second)) - deps.Add(j) - } - - <-deps.Done() - assert.ErrorContains(t, deps.Err(), "failing") - - assert.Equal(t, int32(3), tr.c) -} - -func TestErrorFailSlowMulti(t *testing.T) { - t.Parallel() - - p := NewPool(4) - defer p.Stop(nil) - ctx := context.Background() - - deps := &WaitGroup{} - - for i := 0; i < 3; i++ { - jerr := p.Schedule(ctx, &Job{ - Name: fmt.Sprintf("j%v", i), - Do: func(w *Worker, ctx context.Context) error { - return fmt.Errorf("failing%v", i) - }, - }) - deps.Add(jerr) - } - - <-deps.Done() - t.Log(deps.Err()) - multierr.Errors(deps.Err()) -} - -func TestStress(t *testing.T) { - t.Parallel() - - p := NewPool(1000) - defer p.Stop(nil) - ctx := context.Background() - - tr := &tracker{} - - g1 := &WaitGroup{} - for i := 0; i < 200; i++ { - j := p.Schedule(ctx, job(t, tr, fmt.Sprintf("g1f%v", i), time.Second)) - g1.Add(j) - } - - g2 := &WaitGroup{} - for i := 0; i < 200; i++ { - j := job(t, tr, fmt.Sprintf("g2f%v", i), time.Second) - j.Deps = g1 - j = p.Schedule(ctx, j) - g2.Add(j) - } - - <-g2.Done() - require.NoError(t, g2.Err()) - - assert.Equal(t, int32(400), tr.c) -} - -func TestPause(t *testing.T) { - t.Parallel() - - // Pool only has a single worker - p := NewPool(1) - defer p.Stop(nil) - ctx := context.Background() - - var c int64 - - mj := p.Schedule(ctx, &Job{ - Do: func(w *Worker, ctx context.Context) error { - // Spawn another job - j := p.Schedule(ctx, &Job{ - Do: func(w *Worker, ctx context.Context) error { - atomic.AddInt64(&c, 1) - return nil - }, - }) - - // Wait for that job, this would deadlock without the Wait - Suspend(ctx, func() { - <-j.Wait() - }) - - // Repeat - - j = p.Schedule(ctx, &Job{ - Do: func(w *Worker, ctx context.Context) error { - atomic.AddInt64(&c, 1) - return nil - }, - }) - - Suspend(ctx, func() { - <-j.Wait() - }) - - return nil - }, - }) - - <-mj.Wait() - - assert.Equal(t, int64(2), c) -} diff --git a/worker/worker.go b/worker/worker.go deleted file mode 100644 index 0eb0bf4f..00000000 --- a/worker/worker.go +++ /dev/null @@ -1,463 +0,0 @@ -package worker - -import ( - "context" - "fmt" - "github.com/charmbracelet/lipgloss" - "github.com/hephbuild/heph/log/log" - "github.com/hephbuild/heph/status" - "github.com/hephbuild/heph/utils/xcontext" - "github.com/hephbuild/heph/utils/xpanic" - "github.com/muesli/termenv" - "io" - "runtime/debug" - "sync" - "sync/atomic" - "time" -) - -type JobState int8 - -const ( - StateUnknown JobState = iota - StateScheduled - StateQueued - StateRunning - StateSuccess - StateFailed - StateSkipped - StateSuspended -) - -func (s JobState) IsDone() bool { - return s == StateSuccess || s == StateFailed || s == StateSkipped -} - -func (s JobState) String() string { - switch s { - case StateScheduled: - return "scheduled" - case StateQueued: - return "queued" - case StateRunning: - return "running" - case StateSuccess: - return "success" - case StateFailed: - return "failed" - case StateSkipped: - return "skipped" - case StateSuspended: - return "suspended" - case StateUnknown: - fallthrough - default: - return "unknown" - } -} - -type Hook interface { - Run(*Job) context.Context -} - -type StageHook struct { - OnScheduled func(*Job) context.Context - OnQueued func(*Job) context.Context - OnStart func(*Job) context.Context - OnEnd func(*Job) context.Context -} - -func (h StageHook) Run(j *Job) context.Context { - switch j.State { - case StateScheduled: - if h.OnScheduled != nil { - return h.OnScheduled(j) - } - case StateQueued: - if h.OnQueued != nil { - return h.OnQueued(j) - } - case StateRunning: - if h.OnStart != nil { - return h.OnStart(j) - } - default: - if j.IsDone() { - if h.OnEnd != nil { - return h.OnEnd(j) - } - } - } - return nil -} - -type Job struct { - Name string - ID uint64 - Deps *WaitGroup - Do func(w *Worker, ctx context.Context) error - State JobState - Hook Hook - - status status.Statuser - - ctx context.Context - cancel context.CancelFunc - doneCh chan struct{} - err error - - runCh chan error - suspendCh chan struct{} - suspendedCh chan struct{} - resumeCh chan struct{} - - TimeScheduled time.Time - TimeQueued time.Time - TimeStart time.Time - TimeEnd time.Time - - m sync.Mutex -} - -var stringRenderer = lipgloss.NewRenderer(io.Discard, termenv.WithColorCache(true)) - -func (j *Job) Status(status status.Statuser) { - j.status = status - if s := status.String(stringRenderer); s != "" { - log.Debug(s) - } -} - -func (j *Job) GetStatus() status.Statuser { - s := j.status - if s == nil { - return status.Clear() - } - - return s -} - -func (j *Job) Interactive() bool { - return true -} - -func (j *Job) prepare() (<-chan error, <-chan struct{}, bool) { - j.m.Lock() - defer j.m.Unlock() - - isInit := j.runCh == nil - - if j.runCh == nil { - j.runCh = make(chan error) - } - - j.suspendCh = make(chan struct{}) - - return j.runCh, j.suspendCh, isInit -} - -func (j *Job) suspend() chan struct{} { - j.m.Lock() - - if j.resumeCh != nil { - panic("pause on paused job") - } - - j.resumeCh = make(chan struct{}) - j.suspendedCh = make(chan struct{}) - - close(j.suspendCh) - - return j.resumeCh -} - -func (j *Job) suspendAck(w *Worker) { - defer j.m.Unlock() - - w.CurrentJob = nil - j.State = StateSuspended - - close(j.suspendedCh) -} - -func (j *Job) resume(w *Worker) { - j.m.Lock() - defer j.m.Unlock() - - resumeCh := j.resumeCh - - if resumeCh == nil { - panic("resume on unpaused job") - } - - j.resumeCh = nil - w.CurrentJob = j - j.State = StateRunning - - close(resumeCh) -} - -func (j *Job) run(w *Worker) { - j.ctx = status.ContextWithHandler(j.ctx, j) - - j.TimeStart = time.Now() - j.State = StateRunning - w.CurrentJob = j - - j.RunHook() - err := xpanic.Recover(func() error { - return j.Do(w, j.ctx) - }, xpanic.Wrap(func(err any) error { - return fmt.Errorf("panic in %v: %v => %v", j.Name, err, string(debug.Stack())) - })) - - j.TimeEnd = time.Now() - w.CurrentJob = nil - - j.runCh <- err -} - -func (j *Job) RunHook() { - if h := j.Hook; h != nil { - ctx := j.ctx - jctx := h.Run(j) - if jctx != nil { - ctx = jctx - } - j.ctx = ctx - } -} - -func (j *Job) Ctx() context.Context { - return j.ctx -} - -func (j *Job) Err() error { - return j.err -} - -func (j *Job) Wait() <-chan struct{} { - return j.doneCh -} - -func (j *Job) Done() { - j.m.Lock() - defer j.m.Unlock() - - j.doneWithState(StateSuccess) -} - -func (j *Job) DoneWithErr(err error, state JobState) { - j.m.Lock() - defer j.m.Unlock() - - if state == StateFailed { - log.Errorf("%v finished with err: %v", j.Name, err) - } else { - log.Tracef("%v finished with %v err: %v", j.Name, state, err) - } - - jerr := JobError{ - ID: j.ID, - Name: j.Name, - State: state, - Err: err, - } - j.err = jerr - j.doneWithState(state) -} - -func (j *Job) doneWithState(state JobState) { - j.State = state - close(j.doneCh) - j.RunHook() - j.cancel() -} - -func (j *Job) IsDone() bool { - return j.State.IsDone() -} - -type Worker struct { - CurrentJob *Job -} - -func (w *Worker) GetStatus() status.Statuser { - j := w.CurrentJob - if j == nil { - return status.Clear() - } - - return j.GetStatus() -} - -type Pool struct { - ctx context.Context - cancel func() - Workers []*Worker - - doneCh chan struct{} - o sync.Once - - jobsCh chan *Job - wg sync.WaitGroup - stopped bool - stopErr error - jobs *WaitGroup - m sync.Mutex - idc uint64 -} - -func NewPool(n int) *Pool { - ctx, cancel := context.WithCancel(context.Background()) - - p := &Pool{ - ctx: ctx, - cancel: cancel, - jobsCh: make(chan *Job), - doneCh: make(chan struct{}), - jobs: &WaitGroup{}, - } - - for i := 0; i < n; i++ { - w := &Worker{} - p.Workers = append(p.Workers, w) - - go func() { - for j := range p.jobsCh { - if p.stopped { - // Drain chan - p.finalize(j, fmt.Errorf("pool stopped"), true) - continue - } - - runCh, suspendCh, isInit := j.prepare() - - if isInit { - go func() { - j.run(w) - }() - } else { - j.resume(w) - } - - select { - case err := <-runCh: - p.finalize(j, err, false) - case <-suspendCh: - j.suspendAck(w) - } - } - }() - } - - return p -} - -func (p *Pool) Schedule(ctx context.Context, job *Job) *Job { - p.wg.Add(1) - - ctx, cancel := context.WithCancel(xcontext.CancellableContext{ - Parent: ctx, - Cancel: p.ctx, - }) - - job.ID = atomic.AddUint64(&p.idc, 1) - job.State = StateScheduled - job.TimeScheduled = time.Now() - job.ctx = ContextWithPoolJob(ctx, p, job) - job.cancel = cancel - job.doneCh = make(chan struct{}) - if job.Deps == nil { - job.Deps = &WaitGroup{} - } - - log.Tracef("Scheduling %v %v", job.Name, job.ID) - - p.jobs.Add(job) - job.RunHook() - - go func() { - select { - case <-job.ctx.Done(): - p.finalize(job, job.ctx.Err(), true) - return - case <-job.Deps.Done(): - if err := job.Deps.Err(); err != nil { - p.finalize(job, CollectRootErrors(err), true) - return - } - } - - job.State = StateQueued - job.TimeQueued = time.Now() - job.RunHook() - - p.jobsCh <- job - }() - - return job -} - -func (p *Pool) finalize(job *Job, err error, skippedOnErr bool) { - if err == nil { - job.Done() - //log.Debugf("finalize job: %v %v", job.Name, job.State.String()) - } else { - if skippedOnErr { - job.DoneWithErr(err, StateSkipped) - } else { - job.DoneWithErr(err, StateFailed) - } - //log.Debugf("finalize job err: %v %v: %v", job.Name, job.State.String(), err) - } - - p.wg.Done() -} - -func (p *Pool) Jobs() []*Job { - return p.jobs.jobs[:] -} - -func (p *Pool) IsDone() bool { - return p.jobs.IsDone() -} - -func (p *Pool) Done() <-chan struct{} { - p.m.Lock() - defer p.m.Unlock() - - p.o.Do(func() { - p.doneCh = make(chan struct{}) - - go func() { - p.wg.Wait() - - p.m.Lock() - defer p.m.Unlock() - - close(p.doneCh) - p.o = sync.Once{} - }() - }) - - return p.doneCh -} - -func (p *Pool) Err() error { - return p.stopErr -} - -func (p *Pool) Stop(err error) { - p.m.Lock() - defer p.m.Unlock() - - if p.stopped { - return - } - - p.stopped = true - p.stopErr = err - - p.cancel() -} diff --git a/worker2/dag.go b/worker2/dag.go new file mode 100644 index 00000000..08be41ab --- /dev/null +++ b/worker2/dag.go @@ -0,0 +1,241 @@ +package worker2 + +import ( + "fmt" + "github.com/hephbuild/heph/utils/ads" + "github.com/hephbuild/heph/utils/sets" + "strings" + "sync" +) + +type nodesTransitive[T any] struct { + m sync.RWMutex + nodes *sets.Set[*Node[T], *Node[T]] + transitiveNodes *sets.Set[*Node[T], *Node[T]] + transitiveDirty bool + transitiveGetter func(d *Node[T]) *nodesTransitive[T] + transitiveReverse bool +} + +func (d *nodesTransitive[T]) Add(dep *Node[T]) { + d.m.Lock() + defer d.m.Unlock() + + d.nodes.Add(dep) + d.transitiveDirty = true +} + +func (d *nodesTransitive[T]) MarkTransitiveDirty() { + d.m.Lock() + defer d.m.Unlock() + + d.transitiveDirty = true +} + +func (d *nodesTransitive[T]) MarkTransitiveInvalid() { + d.m.Lock() + defer d.m.Unlock() + + d.transitiveNodes = nil +} + +func (d *nodesTransitive[T]) Remove(dep *Node[T]) { + d.m.Lock() + defer d.m.Unlock() + + d.nodes.Remove(dep) + d.transitiveNodes = nil +} + +func (d *nodesTransitive[T]) Has(dep *Node[T]) bool { + d.m.RLock() + defer d.m.RUnlock() + + return d.nodes.Has(dep) +} + +func (d *nodesTransitive[T]) Set() *sets.Set[*Node[T], *Node[T]] { + d.m.RLock() + defer d.m.RUnlock() + + return d.nodes +} + +func (d *nodesTransitive[T]) TransitiveSet() *sets.Set[*Node[T], *Node[T]] { + d.m.Lock() + defer d.m.Unlock() + + if d.transitiveNodes == nil { + d.transitiveNodes = d.computeTransitive(true) + } else if d.transitiveDirty { + d.transitiveNodes = d.computeTransitive(false) + } + d.transitiveDirty = false + + return d.transitiveNodes +} + +func (d *nodesTransitive[T]) TransitiveValues() []T { + return ads.Map(d.TransitiveSet().Slice(), func(t *Node[T]) T { + return t.V + }) +} + +func (d *nodesTransitive[T]) Values() []T { + return ads.Map(d.Set().Slice(), func(t *Node[T]) T { + return t.V + }) +} + +func (d *nodesTransitive[T]) computeTransitive(full bool) *sets.Set[*Node[T], *Node[T]] { + s := d.transitiveNodes + if full { + s = sets.NewIdentitySet[*Node[T]](0) + } + for _, dep := range d.nodes.Slice() { + transitive := d.transitiveGetter(dep) + + if d.transitiveReverse { + s.AddAll(transitive.TransitiveSet().Slice()) + s.Add(dep) + } else { + s.Add(dep) + s.AddAll(transitive.TransitiveSet().Slice()) + } + } + return s +} + +func newNodesTransitive[T any](transitiveGetter func(d *Node[T]) *nodesTransitive[T], transitiveReverse bool) *nodesTransitive[T] { + return &nodesTransitive[T]{ + nodes: sets.NewIdentitySet[*Node[T]](0), + transitiveNodes: sets.NewIdentitySet[*Node[T]](0), + transitiveGetter: transitiveGetter, + transitiveReverse: transitiveReverse, + } +} + +type Node[T any] struct { + V T + ID string + frozen bool + m sync.Mutex + + Dependencies *nodesTransitive[T] + Dependees *nodesTransitive[T] +} + +func NewNode[T any](id string, v T) *Node[T] { + return &Node[T]{ + ID: id, + V: v, + Dependencies: newNodesTransitive[T](func(d *Node[T]) *nodesTransitive[T] { + return d.Dependencies + }, false), + Dependees: newNodesTransitive[T](func(d *Node[T]) *nodesTransitive[T] { + return d.Dependees + }, true), + } +} + +func (d *Node[T]) GetID() string { + return d.ID +} + +func (d *Node[T]) AddDependency(deps ...*Node[T]) { + d.m.Lock() + defer d.m.Unlock() + + if d.frozen { + panic("add: frozen") + } + + for _, dep := range deps { + d.addDependency(dep) + } +} + +func (d *Node[T]) addDependency(dep *Node[T]) { + if !d.Dependencies.Has(dep) { + if dep.Dependencies.TransitiveSet().Has(d) { + panic("cycle") + } + + d.Dependencies.Add(dep) + + for _, dependee := range d.Dependees.TransitiveSet().Slice() { + dependee.Dependencies.MarkTransitiveDirty() + } + + if !dep.Dependees.Has(d) { + dep.Dependees.Add(d) + + for _, dep := range dep.Dependencies.TransitiveSet().Slice() { + dep.Dependees.MarkTransitiveDirty() + } + } + } +} + +func (d *Node[T]) RemoveDependency(dep *Node[T]) { + d.m.Lock() + defer d.m.Unlock() + + if d.frozen { + panic("remove: deps is frozen") + } + + if d.Dependencies.Has(dep) { + d.Dependencies.Remove(dep) + + for _, dependee := range d.Dependees.TransitiveSet().Slice() { + dependee.Dependencies.MarkTransitiveInvalid() + } + } + + if dep.Dependees.Has(d) { + dep.Dependees.Remove(d) + + for _, dep := range dep.Dependencies.TransitiveSet().Slice() { + dep.Dependees.MarkTransitiveInvalid() + } + } +} + +func (d *Node[T]) IsFrozen() bool { + d.m.Lock() + defer d.m.Unlock() + + return d.frozen +} + +// Freeze assumes the lock is already held +func (d *Node[T]) Freeze() { + if d.frozen { + return + } + + for _, dep := range d.Dependencies.nodes.Slice() { + if !dep.IsFrozen() { + panic(fmt.Sprintf("attempting to freeze '%v' while all deps aren't frozen, '%v' isnt", d.ID, dep.ID)) + } + } + + d.frozen = true +} + +func (d *Node[T]) DebugString() string { + var sb strings.Builder + fmt.Fprintf(&sb, "%v:\n", d.ID) + deps := ads.Map(d.Dependencies.Set().Slice(), (*Node[T]).GetID) + tdeps := ads.Map(d.Dependencies.TransitiveSet().Slice(), (*Node[T]).GetID) + fmt.Fprintf(&sb, " deps: %v\n", deps) + fmt.Fprintf(&sb, " tdeps: %v\n", tdeps) + + depdees := ads.Map(d.Dependees.Set().Slice(), (*Node[T]).GetID) + tdepdees := ads.Map(d.Dependees.TransitiveSet().Slice(), (*Node[T]).GetID) + fmt.Fprintf(&sb, " depdees: %v\n", depdees) + fmt.Fprintf(&sb, " tdepdees: %v\n", tdepdees) + + return sb.String() +} diff --git a/worker2/dep.go b/worker2/dep.go new file mode 100644 index 00000000..dd181fa1 --- /dev/null +++ b/worker2/dep.go @@ -0,0 +1,349 @@ +package worker2 + +import ( + "context" + "github.com/hephbuild/heph/utils/xtypes" + "sync" + "time" +) + +type Dep interface { + GetName() string + Exec(ctx context.Context, ins InStore, outs OutStore) error + GetNode() *Node[Dep] + AddDep(...Dep) + GetHooks() []Hook + Wait() <-chan struct{} + DeepDo(f func(Dep)) + GetCtx() context.Context + SetCtx(ctx context.Context) + GetErr() error + GetState() ExecState + GetScheduledAt() time.Time + GetStartedAt() time.Time + GetQueuedAt() time.Time + + setExecution(*Execution) + getExecution() *Execution + getNamed() map[string]Dep + getMutex() *sync.RWMutex + GetScheduler() Scheduler + GetRequest() map[string]float64 + GetExecutionDebugString() string +} + +func newBase() baseDep { + return baseDep{ + named: map[string]Dep{}, + } +} + +type baseDep struct { + execution *Execution + m sync.RWMutex + node *Node[Dep] + named map[string]Dep + + executionPresentCh chan struct{} + o sync.Once +} + +func (a *baseDep) init() { + if a.executionPresentCh == nil { + a.executionPresentCh = make(chan struct{}) + } +} + +func (a *baseDep) GetNode() *Node[Dep] { + return a.node +} + +func (a *baseDep) AddDep(deps ...Dep) { + for _, dep := range deps { + if xtypes.IsNil(dep) { + continue + } + if named, ok := dep.(Named); ok { + a.named[named.Name] = dep + dep = named.Dep + } + a.GetNode().AddDependency(dep.GetNode()) + } +} + +func (a *baseDep) getNamed() map[string]Dep { + if !a.GetNode().frozen { + panic("not frozen") + } + return a.named +} + +func (a *baseDep) setExecution(e *Execution) { + a.o.Do(a.init) + + if a.execution != nil { + if a.execution != e { + panic("trying to assign different execution to a Dep") + } + return + } + + a.execution = e + close(a.executionPresentCh) +} + +func (a *baseDep) getExecution() *Execution { + a.o.Do(a.init) + + return a.execution +} + +func (a *baseDep) GetExecutionDebugString() string { + exec := a.getExecution() + if exec == nil { + return "" + } + + return exec.debugString +} + +func (a *baseDep) Wait() <-chan struct{} { + a.o.Do(a.init) + + if exec := a.execution; exec != nil { + return exec.Wait() + } + + // Allow to wait Dep that is not scheduled yet + + doneCh := make(chan struct{}) + + go func() { + <-a.executionPresentCh + <-a.execution.Wait() + close(doneCh) + }() + + return doneCh +} + +func (a *baseDep) getMutex() *sync.RWMutex { + return &a.m +} + +func (a *baseDep) GetErr() error { + exec := a.execution + if exec == nil { + return nil + } + + return exec.Err +} + +func (a *baseDep) GetState() ExecState { + exec := a.execution + if exec == nil { + return ExecStateUnknown + } + + return exec.State +} + +func (a *baseDep) GetScheduledAt() time.Time { + exec := a.execution + if exec == nil { + return time.Time{} + } + + return exec.ScheduledAt +} + +func (a *baseDep) GetStartedAt() time.Time { + exec := a.execution + if exec == nil { + return time.Time{} + } + + return exec.StartedAt +} + +func (a *baseDep) GetQueuedAt() time.Time { + exec := a.execution + if exec == nil { + return time.Time{} + } + + return exec.QueuedAt +} + +type ActionConfig struct { + Ctx context.Context + Name string + Deps []Dep + Hooks []Hook + Scheduler Scheduler + Requests map[string]float64 + Do func(ctx context.Context, ins InStore, outs OutStore) error +} + +type Action struct { + baseDep + ctx context.Context + name string + hooks []Hook + scheduler Scheduler + requests map[string]float64 + do func(ctx context.Context, ins InStore, outs OutStore) error +} + +func (a *Action) GetScheduler() Scheduler { + return a.scheduler +} + +func (a *Action) GetRequest() map[string]float64 { + return a.requests +} + +func (a *Action) GetName() string { + return a.name +} + +func (a *Action) GetCtx() context.Context { + if ctx := a.ctx; ctx != nil { + return ctx + } + return context.Background() +} + +func (a *Action) SetCtx(ctx context.Context) { + a.ctx = ctx +} + +func (a *Action) OutputCh() <-chan Value { + h, ch := OutputHook() + a.hooks = append(a.hooks, h) + return ch +} + +func (a *Action) ErrorCh() <-chan error { + h, ch := ErrorHook() + a.hooks = append(a.hooks, h) + return ch +} + +func (a *Action) GetHooks() []Hook { + return a.hooks +} + +func (a *Action) Exec(ctx context.Context, ins InStore, outs OutStore) error { + if a.do == nil { + return nil + } + return a.do(ctx, ins, outs) +} + +func (a *Action) DeepDo(f func(Dep)) { + deepDo(a, f) +} + +type GroupConfig struct { + Name string + Deps []Dep +} + +type Group struct { + baseDep + name string +} + +func (g *Group) GetScheduler() Scheduler { return nil } + +func (g *Group) GetName() string { + return g.name +} + +func (g *Group) GetHooks() []Hook { + return nil +} + +func (g *Group) DeepDo(f func(Dep)) { + deepDo(g, f) +} + +func (g *Group) GetRequest() map[string]float64 { + return nil +} + +func (g *Group) SetCtx(ctx context.Context) { + // TODO +} + +func (g *Group) GetCtx() context.Context { + return context.Background() +} + +func (g *Group) Exec(ctx context.Context, ins InStore, outs OutStore) error { + e := executionFromContext(ctx) + outs.Set(MapValue(e.inputs)) + return nil +} + +type Named struct { + Name string + Dep +} + +func Serial(deps []Dep) Dep { + out := deps[0] + + for i, dep := range deps { + if i == 0 { + continue + } + + prev := out + dep.AddDep(prev) + out = dep + } + + return out +} + +func NewChanDep[T any](ctx context.Context, ch chan T) Dep { + return NewAction(ActionConfig{ + Ctx: ctx, + Do: func(ctx context.Context, ins InStore, outs OutStore) error { + return WaitChan(ctx, ch) + }, + }) +} + +func NewSemDep(ctx context.Context, name string) *Sem { + wg := &sync.WaitGroup{} + return &Sem{ + Dep: NewAction(ActionConfig{ + Ctx: ctx, + Name: name, + Do: func(ctx context.Context, ins InStore, outs OutStore) error { + return WaitE(ctx, func() error { + wg.Wait() + return ctx.Err() + }) + }, + }), + wg: wg, + } +} + +type Sem struct { + Dep + wg *sync.WaitGroup +} + +func (s *Sem) AddSem(delta int) { + s.wg.Add(delta) +} + +func (s *Sem) DoneSem() { + s.wg.Done() +} diff --git a/worker2/dep_action.go b/worker2/dep_action.go new file mode 100644 index 00000000..f03d096d --- /dev/null +++ b/worker2/dep_action.go @@ -0,0 +1,28 @@ +package worker2 + +type EventDeclared struct { + Dep Dep +} + +func NewAction(cfg ActionConfig) *Action { + a := &Action{baseDep: newBase()} + a.node = NewNode[Dep](cfg.Name, a) + + a.name = cfg.Name + a.ctx = cfg.Ctx + a.name = cfg.Name + a.AddDep(cfg.Deps...) + a.hooks = cfg.Hooks + a.scheduler = cfg.Scheduler + a.requests = cfg.Requests + a.do = cfg.Do + + for _, hook := range a.hooks { + if hook == nil { + continue + } + hook(EventDeclared{Dep: a}) + } + + return a +} diff --git a/worker2/dep_group.go b/worker2/dep_group.go new file mode 100644 index 00000000..0c5b0918 --- /dev/null +++ b/worker2/dep_group.go @@ -0,0 +1,19 @@ +package worker2 + +func NewGroup(deps ...Dep) *Group { + return NewNamedGroup("", deps...) +} + +func NewNamedGroup(name string, deps ...Dep) *Group { + return NewGroupWith(GroupConfig{Name: name, Deps: deps}) +} + +func NewGroupWith(cfg GroupConfig) *Group { + g := &Group{baseDep: newBase()} + g.node = NewNode[Dep](cfg.Name, g) + + g.name = cfg.Name + g.AddDep(cfg.Deps...) + + return g +} diff --git a/worker2/dep_utils.go b/worker2/dep_utils.go new file mode 100644 index 00000000..ef36d5b8 --- /dev/null +++ b/worker2/dep_utils.go @@ -0,0 +1,66 @@ +package worker2 + +import "sync/atomic" + +func deepDo(a Dep, f func(Dep)) { + f(a) + for _, dep := range a.GetNode().Dependencies.TransitiveValues() { + f(dep) + } +} + +type Stats struct { + All uint64 + Completed uint64 + + Scheduled uint64 + Waiting uint64 + Succeeded uint64 + Failed uint64 + Skipped uint64 + Suspended uint64 + Running uint64 +} + +func (s *Stats) record(dep Dep) { + atomic.AddUint64(&s.All, 1) + + j := dep.getExecution() + if j == nil { + return + } + + if j.State.IsFinal() { + atomic.AddUint64(&s.Completed, 1) + } + + switch j.State { + case ExecStateQueued: + atomic.AddUint64(&s.Waiting, 1) + case ExecStateSucceeded: + atomic.AddUint64(&s.Succeeded, 1) + case ExecStateFailed: + atomic.AddUint64(&s.Failed, 1) + case ExecStateSkipped: + atomic.AddUint64(&s.Skipped, 1) + case ExecStateSuspended: + atomic.AddUint64(&s.Suspended, 1) + case ExecStateRunning: + atomic.AddUint64(&s.Running, 1) + case ExecStateScheduled: + atomic.AddUint64(&s.Scheduled, 1) + } +} + +func CollectStats(dep Dep) Stats { + s := Stats{} + dep.DeepDo(func(dep Dep) { + if _, ok := dep.(*Group); ok { + return + } + + s.record(dep) + }) + + return s +} diff --git a/worker2/deps_test.go b/worker2/deps_test.go new file mode 100644 index 00000000..de917063 --- /dev/null +++ b/worker2/deps_test.go @@ -0,0 +1,136 @@ +package worker2 + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "strings" + "testing" +) + +func s(s string) string { + return strings.TrimSpace(s) + "\n" +} + +func TestLink(t *testing.T) { + d1 := NewAction(ActionConfig{Name: "1"}) + d2 := NewAction(ActionConfig{Name: "2", Deps: []Dep{d1}}) + + d3 := NewAction(ActionConfig{Name: "3"}) + d4 := NewAction(ActionConfig{Name: "4", Deps: []Dep{d3}}) + + assertDetached := func() { + assert.Equal(t, s(` +1: + deps: [] + tdeps: [] + depdees: [2] + tdepdees: [2] +`), d1.GetNode().DebugString()) + + assert.Equal(t, s(` +2: + deps: [1] + tdeps: [1] + depdees: [] + tdepdees: [] +`), d2.GetNode().DebugString()) + + assert.Equal(t, s(` +3: + deps: [] + tdeps: [] + depdees: [4] + tdepdees: [4] +`), d3.GetNode().DebugString()) + + assert.Equal(t, s(` +4: + deps: [3] + tdeps: [3] + depdees: [] + tdepdees: [] +`), d4.GetNode().DebugString()) + } + + assertDetached() + + d3.AddDep(d2) + + assert.Equal(t, s(` +1: + deps: [] + tdeps: [] + depdees: [2] + tdepdees: [2 4 3] +`), d1.GetNode().DebugString()) + + assert.Equal(t, s(` +2: + deps: [1] + tdeps: [1] + depdees: [3] + tdepdees: [4 3] +`), d2.GetNode().DebugString()) + + assert.Equal(t, s(` +3: + deps: [2] + tdeps: [2 1] + depdees: [4] + tdepdees: [4] +`), d3.GetNode().DebugString()) + + assert.Equal(t, s(` +4: + deps: [3] + tdeps: [3 2 1] + depdees: [] + tdepdees: [] +`), d4.GetNode().DebugString()) + + d3.GetNode().RemoveDependency(d2.GetNode()) + + assertDetached() +} + +func TestCycle1(t *testing.T) { + d1 := NewAction(ActionConfig{Name: "1"}) + d2 := NewAction(ActionConfig{Name: "2", Deps: []Dep{d1}}) + d3 := NewAction(ActionConfig{Name: "3", Deps: []Dep{d2}}) + + assert.PanicsWithValue(t, "cycle", func() { + d2.AddDep(d3) + }) +} + +func TestCycle2(t *testing.T) { + d1 := NewAction(ActionConfig{Name: "1", Deps: []Dep{ /* d4 */ }}) + d2 := NewAction(ActionConfig{Name: "2", Deps: []Dep{d1}}) + + d3 := NewAction(ActionConfig{Name: "3"}) + d4 := NewAction(ActionConfig{Name: "4", Deps: []Dep{d3}}) + + d1.AddDep(d4) + + assert.PanicsWithValue(t, "cycle", func() { + d3.AddDep(d2) + }) +} + +func TestRemoveStress(t *testing.T) { + root := NewAction(ActionConfig{Name: "root", Deps: []Dep{}}) + + for i := 0; i < 1000; i++ { + d := NewAction(ActionConfig{Name: fmt.Sprint(i)}) + root.AddDep(d) + + for j := 0; j < 1000; j++ { + d1 := NewAction(ActionConfig{Name: fmt.Sprintf("%v-%v", i, j)}) + d.AddDep(d1) + } + } + + group := NewAction(ActionConfig{Name: "group", Deps: []Dep{root}}) + + group.GetNode().RemoveDependency(root.GetNode()) +} diff --git a/worker2/engine.go b/worker2/engine.go new file mode 100644 index 00000000..40a1d521 --- /dev/null +++ b/worker2/engine.go @@ -0,0 +1,396 @@ +package worker2 + +import ( + "github.com/bep/debounce" + "github.com/dlsniper/debugger" + "github.com/hephbuild/heph/utils/ads" + "go.uber.org/multierr" + "runtime" + "sync" + "sync/atomic" + "time" +) + +type Engine struct { + execUid uint64 + wg sync.WaitGroup + defaultScheduler Scheduler + workers []*Worker + m sync.RWMutex + eventsCh chan Event + hooks []Hook +} + +func NewEngine() *Engine { + e := &Engine{ + eventsCh: make(chan Event, 1000), + defaultScheduler: UnlimitedScheduler{}, + } + + return e +} + +func (e *Engine) SetDefaultScheduler(s Scheduler) { + e.defaultScheduler = s +} + +func (e *Engine) GetWorkers() []*Worker { + e.m.Lock() + defer e.m.Unlock() + + return e.workers[:] +} + +func (e *Engine) loop() { + for event := range e.eventsCh { + e.handle(event) + } +} + +func (e *Engine) handle(event Event) { + switch event := event.(type) { + case EventReady: + e.runHooks(event, event.Execution) + e.start(event.Execution) + case EventSkipped: + e.finalize(event.Execution, ExecStateSkipped, event.Error) + e.runHooks(event, event.Execution) + case EventCompleted: + if event.Error != nil { + e.finalize(event.Execution, ExecStateFailed, event.Error) + } else { + e.finalize(event.Execution, ExecStateSucceeded, nil) + } + e.runHooks(event, event.Execution) + default: + if event, ok := event.(WithExecution); ok { + defer e.runHooks(event, event.getExecution()) + } + } +} + +func (e *Engine) finalize(exec *Execution, state ExecState, err error) { + exec.m.Lock() + exec.Err = err + exec.State = state + close(exec.completedCh) + exec.m.Unlock() + + for _, dep := range exec.Dep.GetNode().Dependees.Values() { + dexec := e.executionForDep(dep) + + dexec.broadcast() + } + + e.wg.Done() +} + +func (e *Engine) runHooks(event Event, exec *Execution) { + for _, hook := range e.hooks { + if hook == nil { + continue + } + hook(event) + } + + for _, hook := range exec.Dep.GetHooks() { + if hook == nil { + continue + } + hook(event) + } +} + +func (e *Engine) waitForDeps(exec *Execution) error { + exec.c.L.Lock() + defer exec.c.L.Unlock() + + var errs []error + for { + depObj := exec.Dep.GetNode() + + allDepsSucceeded := true + allDepsDone := true + errs = errs[:0] + for _, dep := range depObj.Dependencies.Values() { + depExec := e.scheduleOne(dep) + + depExec.m.Lock() + state := depExec.State + depExec.m.Unlock() + + if !state.IsFinal() { + allDepsDone = false + } + + if state != ExecStateSucceeded { + allDepsSucceeded = false + } + + switch state { + case ExecStateSkipped, ExecStateFailed: + errs = append(errs, Error{ + ID: depExec.ID, + State: depExec.State, + Name: depExec.Dep.GetName(), + Err: depExec.Err, + }) + } + } + + if allDepsDone { + if len(errs) > 0 { + return multierr.Combine(errs...) + } + + if allDepsSucceeded { + if e.tryFreeze(depObj) { + return nil + } + } + } + + exec.c.Wait() + } +} + +func (e *Engine) tryFreeze(depObj *Node[Dep]) bool { + depObj.m.Lock() // prevent any deps modification + defer depObj.m.Unlock() + + for _, dep := range depObj.Dependencies.Values() { + if dep.GetState() != ExecStateSucceeded { + return false + } + } + + depObj.Freeze() + return true +} + +func (e *Engine) waitForDepsAndSchedule(exec *Execution) { + debugger.SetLabels(func() []string { + return []string{ + "where", "waitForDepsAndSchedule", + "dep_id", exec.Dep.GetName(), + } + }) + + err := e.waitForDeps(exec) + if err != nil { + e.notifySkipped(exec, err) + return + } + + exec.m.Lock() + ins := make(map[string]Value, len(exec.Dep.getNamed())) + for name, dep := range exec.Dep.getNamed() { + exec := e.executionForDep(dep) + + vv := exec.outStore.Get() + + ins[name] = vv + } + exec.inputs = ins + exec.State = ExecStateQueued + exec.QueuedAt = time.Now() + exec.scheduler = exec.Dep.GetScheduler() + if exec.scheduler == nil { + if _, ok := exec.Dep.(*Group); ok { + // TODO: change to properly use ResourceScheduler + exec.scheduler = groupScheduler + } else { + exec.scheduler = e.defaultScheduler + } + } + exec.m.Unlock() + + e.queue(exec) +} + +var groupScheduler = NewLimitScheduler(runtime.NumCPU()) + +func (e *Engine) queue(exec *Execution) { + e.notifyQueued(exec) + + err := exec.scheduler.Schedule(exec.Dep, nil) // TODO: pass a way for the scheduler to write into the input + if err != nil { + e.notifyCompleted(exec, nil, err) + return + } + + e.notifyReady(exec) +} + +func (e *Engine) notifySkipped(exec *Execution, err error) { + e.eventsCh <- EventSkipped{ + At: time.Now(), + Error: err, + Execution: exec, + } +} + +func (e *Engine) notifyCompleted(exec *Execution, output Value, err error) { + e.eventsCh <- EventCompleted{ + At: time.Now(), + Execution: exec, + Output: output, + Error: err, + } +} + +func (e *Engine) notifyReady(exec *Execution) { + e.eventsCh <- EventReady{ + At: time.Now(), + Execution: exec, + } +} + +func (e *Engine) notifyQueued(exec *Execution) { + e.eventsCh <- EventQueued{ + At: time.Now(), + Execution: exec, + } +} + +func (e *Engine) start(exec *Execution) { + e.m.Lock() + defer e.m.Unlock() + + w := &Worker{ + ctx: exec.Dep.GetCtx(), + exec: exec, + queue: func() { + e.queue(exec) + }, + } + e.workers = append(e.workers, w) + + go func() { + w.Run() + + e.m.Lock() + defer e.m.Unlock() + + e.workers = ads.Filter(e.workers, func(worker *Worker) bool { + return worker != w + }) + }() + + e.runHooks(EventStarted{Execution: exec}, exec) +} + +func (e *Engine) RegisterHook(hook Hook) { + e.hooks = append(e.hooks, hook) +} + +func (e *Engine) Run() { + debugger.SetLabels(func() []string { + return []string{ + "where", "worker2.Engine.Run", + } + }) + + e.loop() +} + +func (e *Engine) Stop() { + close(e.eventsCh) +} + +func (e *Engine) Wait() <-chan struct{} { + ch := make(chan struct{}) + go func() { + e.wg.Wait() + close(ch) + }() + + return ch +} + +func (e *Engine) executionForDep(dep Dep) *Execution { + if exec := dep.getExecution(); exec != nil { + return exec + } + + return e.registerOne(dep, true) +} + +func (e *Engine) Schedule(a Dep) Dep { + if !a.GetScheduledAt().IsZero() { + return nil + } + + for _, dep := range a.GetNode().Dependencies.Values() { + _ = e.Schedule(dep) + } + + _ = e.scheduleOne(a) + + return a +} + +func (e *Engine) registerOne(dep Dep, lock bool) *Execution { + m := dep.getMutex() + if lock { + m.Lock() + defer m.Unlock() + } + + if exec := dep.getExecution(); exec != nil { + return exec + } + + exec := &Execution{ + ID: atomic.AddUint64(&e.execUid, 1), + Dep: dep, + outStore: &outStore{}, + eventsCh: e.eventsCh, + completedCh: make(chan struct{}), + m: m, + + // see field comments + errCh: nil, + inputs: nil, + } + debounceBroadcast := debounce.New(time.Millisecond) + exec.broadcast = func() { + debounceBroadcast(func() { + debugger.SetLabels(func() []string { + return []string{ + "where", "debounceBroadcast", + } + }) + + exec.c.L.Lock() + exec.c.Broadcast() + exec.c.L.Unlock() + }) + } + exec.c = sync.NewCond(exec.m) + //exec.c = sync.NewCond(&exec.Dep.GetNode().m) + dep.setExecution(exec) + + return exec +} + +func (e *Engine) scheduleOne(dep Dep) *Execution { + m := dep.getMutex() + m.Lock() + defer m.Unlock() + + exec := e.registerOne(dep, false) + + if exec.ScheduledAt.IsZero() { + e.wg.Add(1) + + exec.ScheduledAt = time.Now() + exec.State = ExecStateScheduled + + e.runHooks(EventScheduled{Execution: exec}, exec) + + go e.waitForDepsAndSchedule(exec) + } + + return exec +} diff --git a/worker2/engine_test.go b/worker2/engine_test.go new file mode 100644 index 00000000..a96fb29a --- /dev/null +++ b/worker2/engine_test.go @@ -0,0 +1,745 @@ +package worker2 + +import ( + "context" + "errors" + "fmt" + "github.com/hephbuild/heph/status" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" +) + +// Number of actions to be processed during a stress test +const StressN = 100000 + +func TestExecSimple(t *testing.T) { + t.Parallel() + + didRun := false + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + didRun = true + fmt.Println("Running 1") + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + <-e.Wait() + <-a.Wait() + + assert.True(t, didRun) +} + +func TestExecSerial(t *testing.T) { + t.Parallel() + n := 500 + + values := make([]int, 0, n) + expected := make([]int, 0, n) + deps := make([]Dep, 0, n) + + for i := 0; i < n; i++ { + i := i + expected = append(expected, i) + a := NewAction(ActionConfig{ + Name: fmt.Sprint(i), + Do: func(ctx context.Context, ins InStore, outs OutStore) error { + values = append(values, i) + return nil + }, + }) + deps = append(deps, a) + } + + serial := Serial(deps) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(serial) + + <-serial.Wait() + + assert.EqualValues(t, expected, values) +} + +func TestDependOnImplicitlyScheduledGroup(t *testing.T) { + t.Parallel() + + g1 := NewGroup() + + a1 := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 1") + return nil + }, + }) + + g1.AddDep(a1) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a1) + + // Group waited on needs to be explicitly scheduled, we can get smarter in the future, + // going up in the tree to find the engine and auto-schedule? Not sure if worth the effort + e.Schedule(g1) + + <-g1.Wait() + <-e.Wait() +} + +func TestStatus(t *testing.T) { + t.Parallel() + + emittedCh := make(chan struct{}) + resumeCh := make(chan struct{}) + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + status.Emit(ctx, status.String("hello")) + close(emittedCh) + <-resumeCh + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + <-emittedCh + + var emittedStatus status.Statuser + for _, worker := range e.GetWorkers() { + emittedStatus = worker.status + if emittedStatus != nil { + break + } + } + require.NotNil(t, emittedStatus) + assert.Equal(t, "hello", emittedStatus.String(nil)) + + close(resumeCh) + + <-a.Wait() +} + +func TestExecHook(t *testing.T) { + t.Parallel() + + ch := make(chan Event, 1000) + a := NewAction(ActionConfig{ + Hooks: []Hook{ + func(event Event) { + ch <- event + }, + }, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 1") + os.Set(NewValue(1)) + return nil + }, + }) + + outputCh := a.OutputCh() + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + <-e.Wait() + close(ch) + + events := make([]string, 0) + for event := range ch { + events = append(events, fmt.Sprintf("%T", event)) + } + + assert.EqualValues(t, []string{"worker2.EventDeclared", "worker2.EventScheduled", "worker2.EventQueued", "worker2.EventReady", "worker2.EventStarted", "worker2.EventCompleted"}, events) + v, _ := (<-outputCh).Get() + assert.Equal(t, int(1), v) +} + +func TestExecError(t *testing.T) { + t.Parallel() + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return fmt.Errorf("beep bop") + }, + }) + + errCh := a.ErrorCh() + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + <-a.Wait() + + assert.ErrorContains(t, <-errCh, "beep bop") +} + +func TestExecErrorSkip(t *testing.T) { + t.Parallel() + a1 := NewAction(ActionConfig{ + Name: "a1", + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return fmt.Errorf("beep bop") + }, + }) + + a2 := NewAction(ActionConfig{ + Name: "a2", + Deps: []Dep{a1}, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return nil + }, + }) + + a3 := NewAction(ActionConfig{ + Name: "a3", + Deps: []Dep{a2}, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return nil + }, + }) + + err1Ch := a1.ErrorCh() + err2Ch := a2.ErrorCh() + err3Ch := a3.ErrorCh() + + e := NewEngine() + e.RegisterHook(LogHook()) + + go e.Run() + defer e.Stop() + + e.Schedule(a3) + + <-a3.Wait() + + err1 := <-err1Ch + err2 := <-err2Ch + err3 := <-err3Ch + + assert.Equal(t, err1, errors.New("beep bop")) + assert.Equal(t, err2, Error{ + ID: 1, + Name: "a1", + State: ExecStateFailed, + Err: errors.New("beep bop"), + }) + assert.Equal(t, err3, Error{ + ID: 2, + Name: "a2", + State: ExecStateSkipped, + Err: Error{ + ID: 1, + Name: "a1", + State: ExecStateFailed, + Err: errors.New("beep bop"), + }, + }) +} + +func TestExecErrorSkipStress(t *testing.T) { + t.Parallel() + a1 := NewAction(ActionConfig{ + Name: "a1", + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return fmt.Errorf("beep bop") + }, + }) + + g := NewGroup() + + scheduler := NewLimitScheduler(runtime.NumCPU()) + + type errContainer struct { + ch <-chan error + d Dep + } + + var errChs2 []<-chan error + var errChs3 []errContainer + + for i := 0; i < StressN/100; i++ { + a2 := NewAction(ActionConfig{ + Name: fmt.Sprintf("2-%v", i), + Deps: []Dep{a1}, + Scheduler: scheduler, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return nil + }, + }) + + errChs2 = append(errChs2, a2.ErrorCh()) + + for j := 0; j < 100; j++ { + a3 := NewAction(ActionConfig{ + Name: fmt.Sprintf("3-%v", j), + Deps: []Dep{a2}, + Scheduler: scheduler, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + return nil + }, + }) + g.AddDep(a3) + + errChs3 = append(errChs3, errContainer{ + ch: a3.ErrorCh(), + d: a2, + }) + } + } + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(g) + + <-g.Wait() + + for _, errCh := range errChs2 { + err := <-errCh + + assert.Equal(t, err, Error{ + ID: 1, + Name: "a1", + State: ExecStateFailed, + Err: errors.New("beep bop"), + }) + } + + for _, c := range errChs3 { + err := <-c.ch + + assert.Equal(t, err, Error{ + ID: c.d.getExecution().ID, + Name: c.d.GetName(), + State: ExecStateSkipped, + Err: Error{ + ID: 1, + Name: "a1", + State: ExecStateFailed, + Err: errors.New("beep bop"), + }, + }) + } +} + +func TestExecCancel(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + a := NewAction(ActionConfig{ + Ctx: ctx, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + <-ctx.Done() + return ctx.Err() + }, + }) + + errCh := a.ErrorCh() + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + cancel() + + <-a.Wait() + + err := <-errCh + + assert.ErrorIs(t, err, context.Canceled) +} + +func TestExecDeps(t *testing.T) { + t.Parallel() + a1_1 := NewAction(ActionConfig{ + Name: "a1_1", + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 1") + + os.Set(NewValue(1)) + return nil + }, + }) + a1_2 := NewAction(ActionConfig{ + Name: "a1_2", + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 2") + + os.Set(NewValue("hello, world")) + return nil + }, + }) + + receivedValue := "" + + a2 := NewAction(ActionConfig{ + Name: "result", + Deps: []Dep{ + Named{Name: "v1", Dep: a1_1}, + Named{Name: "v2", Dep: a1_2}, + }, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + v1 := ds.Get("v1") + v2 := ds.Get("v2") + + fmt.Println("Got values", v1, v2) + + receivedValue = fmt.Sprintf("%v %v", v1, v2) + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a2) + + <-a2.Wait() + + assert.Equal(t, "1 hello, world", receivedValue) +} + +func TestExecGroup(t *testing.T) { + t.Parallel() + a1_1 := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 1") + + os.Set(NewValue(1)) + return nil + }, + }) + a1_2 := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running 2") + + os.Set(NewValue("hello, world")) + return nil + }, + }) + + g := NewGroup( + Named{Name: "v1", Dep: a1_1}, + Named{Name: "v2", Dep: a1_2}, + ) + + var received any + a := NewAction(ActionConfig{ + Deps: []Dep{Named{Name: "v", Dep: g}}, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + received = ds.Get("v") + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + <-a.Wait() + + assert.Equal(t, map[string]any{"v1": 1, "v2": "hello, world"}, received) +} + +func TestExecStressDeep(t *testing.T) { + t.Parallel() + scheduler := NewLimitScheduler(runtime.NumCPU()) + + current := NewGroup() + for l := 0; l < 5000; l++ { + g := NewGroup() + for i := 0; i < 100; i++ { + g.AddDep(NewGroup()) + } + current.AddDep(g) + current = g + } + + e := NewEngine() + e.SetDefaultScheduler(scheduler) + + go e.Run() + defer e.Stop() + + e.Schedule(current) + + <-current.Wait() +} + +func TestExecStress(t *testing.T) { + t.Parallel() + scheduler := NewLimitScheduler(runtime.NumCPU()) + + g := NewGroup() + + n := StressN + + for i := 0; i < n; i++ { + i := i + a := NewAction(ActionConfig{ + Scheduler: scheduler, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + os.Set(NewValue(i)) + return nil + }, + }) + + g.AddDep(Named{Name: fmt.Sprint(i), Dep: a}) + } + + var received any + a := NewAction(ActionConfig{ + Deps: []Dep{Named{Name: "v", Dep: g}}, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + received = ds.Get("v") + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + totalDeps := uint64(n + 1) + + stats1 := CollectStats(a) + assert.Equal(t, Stats{All: totalDeps}, stats1) + + e.Schedule(a) + + <-a.Wait() + + stats3 := CollectStats(a) + assert.Equal(t, Stats{All: totalDeps, Completed: totalDeps, Succeeded: totalDeps}, stats3) + + expected := map[string]any{} + for i := 0; i < n; i++ { + expected[fmt.Sprint(i)] = i + } + + assert.Equal(t, expected, received) +} + +func TestExecProducerConsumer(t *testing.T) { + g := NewGroup() + + n := 10000 + + producer := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running producer") + + for i := 0; i < n; i++ { + i := i + + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + //fmt.Println("Running inner", i) + os.Set(NewValue(i)) + return nil + }, + }) + + g.AddDep(Named{Name: fmt.Sprint(i), Dep: a}) + } + return nil + }, + }) + + g.AddDep(producer) + + var received any + consumer := NewAction(ActionConfig{ + Deps: []Dep{producer, Named{Name: "v", Dep: g}}, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + fmt.Println("Running consumer") + + received = ds.Get("v") + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(consumer) + + <-consumer.Wait() + + expected := map[string]any{} + for i := 0; i < n; i++ { + expected[fmt.Sprint(i)] = i + } + + assert.Equal(t, expected, received) +} + +func TestSuspend(t *testing.T) { + t.Parallel() + logCh := make(chan string) + log := func(s string) { + fmt.Println(s) + logCh <- s + } + resumeCh := make(chan struct{}) + resumeAckCh := make(chan struct{}) + eventCh := make(chan Event, 1000) + a := NewAction(ActionConfig{ + Hooks: []Hook{ + func(event Event) { + eventCh <- event + }, + }, + Do: func(ctx context.Context, ds InStore, os OutStore) error { + log("enter") + defer log("leave") + Wait(ctx, func() { + log("start_wait") + <-resumeCh + resumeAckCh <- struct{}{} + log("end_wait") + }) + return nil + }, + }) + + e := NewEngine() + + go e.Run() + defer e.Stop() + + e.Schedule(a) + + assert.Equal(t, "enter", <-logCh) + assert.Equal(t, "start_wait", <-logCh) + close(resumeCh) + <-resumeAckCh + assert.Equal(t, "end_wait", <-logCh) + assert.Equal(t, "leave", <-logCh) + + <-e.Wait() + close(eventCh) + + events := make([]string, 0) + for event := range eventCh { + events = append(events, fmt.Sprintf("%T", event)) + } + assert.EqualValues(t, []string{"worker2.EventDeclared", "worker2.EventScheduled", "worker2.EventQueued", "worker2.EventReady", "worker2.EventStarted", "worker2.EventSuspended", "worker2.EventQueued", "worker2.EventReady", "worker2.EventStarted", "worker2.EventCompleted"}, events) +} + +func TestSuspendStress(t *testing.T) { + t.Parallel() + + e := NewEngine() + e.SetDefaultScheduler(NewLimitScheduler(100)) + + var wg sync.WaitGroup + var done int64 + for i := 0; i < StressN; i++ { + wg.Add(1) + + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ds InStore, os OutStore) error { + defer wg.Done() + defer atomic.AddInt64(&done, 1) + + Wait(ctx, func() { + time.Sleep(time.Microsecond) + }) + return nil + }, + }) + e.Schedule(a) + } + + go e.Run() + defer e.Stop() + + go func() { + for { + time.Sleep(time.Second) + t.Log(done) + } + }() + + wg.Wait() + time.Sleep(time.Second) // todo figure out why things are trying to send events after the pool is stopped +} + +func TestSuspendLimit(t *testing.T) { + t.Parallel() + + e := NewEngine() + e.SetDefaultScheduler(NewLimitScheduler(1)) + + g := NewGroup() + + for i := 0; i < 100; i++ { + a := NewAction(ActionConfig{ + Do: func(ctx context.Context, ins InStore, outs OutStore) error { + Wait(ctx, func() { + time.Sleep(time.Second) + }) + return nil + }, + }) + g.AddDep(a) + + e.Schedule(a) + } + + e.Schedule(g) + + go e.Run() + defer e.Stop() + + <-g.Wait() + <-e.Wait() +} diff --git a/worker2/error.go b/worker2/error.go new file mode 100644 index 00000000..d5f42163 --- /dev/null +++ b/worker2/error.go @@ -0,0 +1,91 @@ +package worker2 + +import ( + "errors" + "fmt" + "go.uber.org/multierr" +) + +type Error struct { + ID uint64 + Name string + State ExecState + Err error + + root error +} + +func (e Error) Skipped() bool { + return e.State == ExecStateSkipped +} + +func CollectUniqueErrors(inErrs []error) []error { + var errs []error + jerrs := map[uint64]Error{} + + for _, err := range inErrs { + var jerr Error + if errors.As(err, &jerr) { + jerrs[jerr.ID] = jerr + } else { + errs = append(errs, err) + } + } + + for _, err := range jerrs { + errs = append(errs, err) + } + + return errs +} + +func CollectRootErrors(err error) []error { + errs := make([]error, 0) + + for _, err := range multierr.Errors(err) { + var jerr Error + if errors.As(err, &jerr) { + errs = append(errs, jerr.Root()) + } else { + errs = append(errs, err) + } + } + + return CollectUniqueErrors(errs) +} + +func (e Error) Root() error { + if e.root != nil { + return e.root + } + + if !e.Skipped() { + return e + } + + var roots []error + for _, err := range multierr.Errors(e.Err) { + var jerr Error + if errors.As(err, &jerr) { + roots = append(roots, jerr.Root()) + } else { + roots = append(roots, err) + } + } + + if len(roots) == 0 { + e.root = e + return e + } + + e.root = multierr.Combine(roots...) + return e.root +} + +func (e Error) Unwrap() error { + return e.Err +} + +func (e Error) Error() string { + return fmt.Sprintf("%v: %v: %v", e.Name, e.State, e.Err) +} diff --git a/worker2/events.go b/worker2/events.go new file mode 100644 index 00000000..832e3ffb --- /dev/null +++ b/worker2/events.go @@ -0,0 +1,75 @@ +package worker2 + +import "time" + +type Event any + +type WithExecution interface { + getExecution() *Execution +} + +type EventCompleted struct { + At time.Time + Execution *Execution + Output Value + Error error +} + +func (e EventCompleted) getExecution() *Execution { + return e.Execution +} + +type EventScheduled struct { + At time.Time + Execution *Execution +} + +func (e EventScheduled) getExecution() *Execution { + return e.Execution +} + +type EventQueued struct { + At time.Time + Execution *Execution +} + +func (e EventQueued) getExecution() *Execution { + return e.Execution +} + +type EventStarted struct { + At time.Time + Execution *Execution +} + +func (e EventStarted) getExecution() *Execution { + return e.Execution +} + +type EventSkipped struct { + At time.Time + Execution *Execution + Error error +} + +func (e EventSkipped) getExecution() *Execution { + return e.Execution +} + +type EventReady struct { + At time.Time + Execution *Execution +} + +func (e EventReady) getExecution() *Execution { + return e.Execution +} + +type EventSuspended struct { + At time.Time + Execution *Execution +} + +func (e EventSuspended) getExecution() *Execution { + return e.Execution +} diff --git a/worker2/execution.go b/worker2/execution.go new file mode 100644 index 00000000..6ea867e5 --- /dev/null +++ b/worker2/execution.go @@ -0,0 +1,223 @@ +package worker2 + +import ( + "context" + "fmt" + "github.com/dlsniper/debugger" + "runtime/debug" + "strconv" + "sync" + "time" +) + +type ExecState int + +func (s ExecState) IsFinal() bool { + return s == ExecStateSucceeded || s == ExecStateFailed || s == ExecStateSkipped +} +func (s ExecState) String() string { + switch s { + case ExecStateUnknown: + return "Unknown" + case ExecStateScheduled: + return "Scheduled" + case ExecStateQueued: + return "Queued" + case ExecStateRunning: + return "Running" + case ExecStateSucceeded: + return "Succeeded" + case ExecStateFailed: + return "Failed" + case ExecStateSkipped: + return "Skipped" + case ExecStateSuspended: + return "Suspended" + } + + return strconv.Itoa(int(s)) +} + +const ( + ExecStateUnknown ExecState = iota + ExecStateScheduled + ExecStateQueued + ExecStateRunning + ExecStateSucceeded + ExecStateFailed + ExecStateSkipped + ExecStateSuspended +) + +type Execution struct { + ID uint64 + Dep Dep + State ExecState + Err error + outStore OutStore + eventsCh chan Event + c *sync.Cond + broadcast func() + + scheduler Scheduler + + errCh chan error // gets populated when exec is called + inputs map[string]Value // gets populated before marking as ready + m *sync.RWMutex + + suspendCh chan *SuspendBag + resumeAckCh chan struct{} + + completedCh chan struct{} + + ScheduledAt time.Time + StartedAt time.Time + QueuedAt time.Time + + debugString string +} + +func (e *Execution) String() string { + if id := e.Dep.GetName(); id != "" { + return id + } + + return fmt.Sprintf("%p", e) +} + +func (e *Execution) Wait() <-chan struct{} { + return e.completedCh +} + +func (e *Execution) GetOutput() Value { + return e.outStore.Get() +} + +type ErrSuspended struct { + Bag *SuspendBag +} + +func (e ErrSuspended) Error() string { + return "suspended" +} + +func (e *Execution) Run(ctx context.Context) error { + e.m.Lock() + if e.errCh == nil { + e.errCh = make(chan error) + e.suspendCh = make(chan *SuspendBag) + + if !e.StartedAt.IsZero() { + panic("double start detected") + } + + e.StartedAt = time.Now() + + go func() { + err := e.run(ctx) + e.errCh <- err + }() + } else { + e.ResumeAck() + e.State = ExecStateRunning + } + e.m.Unlock() + + select { + case sb := <-e.WaitSuspend(): + e.State = ExecStateSuspended + return ErrSuspended{Bag: sb} + case err := <-e.errCh: + return err + } +} + +func (e *Execution) run(ctx context.Context) error { + debugger.SetLabels(func() []string { + return []string{ + "where", "Execution.run", + "dep_id", e.Dep.GetName(), + } + }) + + if g, ok := e.Dep.(*Group); ok { + return g.Exec(ctx, nil, e.outStore) + } + + ins := &inStore{m: map[string]any{}} + for k, value := range e.inputs { + vv, err := value.Get() + if err != nil { + return fmt.Errorf("%v: %w", k, err) + } + + ins.m[k] = vv + } + + //return e.Dep.Exec(ctx, ins, e.outStore) + + return e.safeExec(ctx, ins) +} + +func (e *Execution) safeExec(ctx context.Context, ins InStore) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v\n%s", r, debug.Stack()) + } + }() + + return e.Dep.Exec(ctx, ins, e.outStore) +} + +type SuspendBag struct { + resumeCh chan struct{} + resumeAckCh chan struct{} +} + +func (e *Execution) Suspend() *SuspendBag { + e.m.Lock() + defer e.m.Unlock() + + // useful if debug poolwait is in use, commented for perf + //stack := debug.Stack() + //e.debugString = string(stack) + + if e.State == ExecStateSuspended { + panic("attempting to suspend an already suspended execution") + } + + resumeCh := make(chan struct{}) + e.resumeAckCh = make(chan struct{}) + sb := &SuspendBag{ + resumeCh: resumeCh, + resumeAckCh: e.resumeAckCh, + } + + e.suspendCh <- sb + + return sb +} + +func (e *SuspendBag) Resume() <-chan struct{} { + ackCh := e.resumeAckCh + + if ackCh == nil { + panic("attempting to resume an unsuspended execution") + } + + close(e.resumeCh) + + return ackCh +} + +func (e *SuspendBag) WaitResume() <-chan struct{} { + return e.resumeCh +} + +func (e *Execution) ResumeAck() { + close(e.resumeAckCh) +} + +func (e *Execution) WaitSuspend() chan *SuspendBag { + return e.suspendCh +} diff --git a/worker2/hook.go b/worker2/hook.go new file mode 100644 index 00000000..a7f40da7 --- /dev/null +++ b/worker2/hook.go @@ -0,0 +1,94 @@ +package worker2 + +import ( + "context" + "fmt" +) + +type Hook func(Event) + +func OutputHook() (Hook, <-chan Value) { + ch := make(chan Value, 1) + return func(event Event) { + switch event := event.(type) { + case EventCompleted: + ch <- event.Output + close(ch) + case EventSkipped: + close(ch) + } + }, ch +} + +func ErrorHook() (Hook, <-chan error) { + ch := make(chan error, 1) + return func(event Event) { + switch event := event.(type) { + case EventCompleted: + ch <- event.Error + close(ch) + case EventSkipped: + ch <- event.Error + close(ch) + } + }, ch +} + +func LogHook() Hook { + return func(event Event) { + if event, ok := event.(WithExecution); ok { + fmt.Printf("%v: %T %+v\n", event.getExecution().Dep.GetName(), event, event) + } else { + fmt.Printf("%T %+v\n", event, event) + } + } +} + +type StageHook struct { + OnScheduled func(Dep) context.Context + // OnWaiting + OnQueued func(Dep) context.Context + OnStart func(Dep) context.Context + OnEnd func(Dep) context.Context +} + +func (h StageHook) Hook() Hook { + return func(event1 Event) { + event, ok := event1.(WithExecution) + if !ok { + return + } + + ctx := h.run(event) + if ctx != nil { + event.getExecution().Dep.SetCtx(ctx) + } + } +} + +func (h StageHook) run(event WithExecution) context.Context { + state := event.getExecution().State + dep := event.getExecution().Dep + + switch state { + case ExecStateScheduled: + if h.OnScheduled != nil { + return h.OnScheduled(dep) + } + case ExecStateQueued: + if h.OnQueued != nil { + return h.OnQueued(dep) + } + case ExecStateRunning: + if h.OnStart != nil { + return h.OnStart(dep) + } + default: + if state.IsFinal() { + if h.OnEnd != nil { + return h.OnEnd(dep) + } + } + } + return nil +} diff --git a/worker/poolui/tui.go b/worker2/poolui/tui.go similarity index 58% rename from worker/poolui/tui.go rename to worker2/poolui/tui.go index f679a218..80a0f52f 100644 --- a/worker/poolui/tui.go +++ b/worker2/poolui/tui.go @@ -6,22 +6,28 @@ import ( tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "github.com/hephbuild/heph/log/log" - "github.com/hephbuild/heph/utils/ads" + "github.com/hephbuild/heph/status" "github.com/hephbuild/heph/utils/xcontext" "github.com/hephbuild/heph/utils/xtea" "github.com/hephbuild/heph/utils/xtime" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "strings" "time" ) +type workerEntry struct { + status status.Statuser + duration time.Duration + exec *worker2.Execution +} + type UpdateMessage struct { - workers []*worker.Worker - stats worker.WaitGroupStats + workers []workerEntry + stats worker2.Stats final bool } -func New(ctx context.Context, name string, deps *worker.WaitGroup, pool *worker.Pool, quitWhenDone bool) *Model { +func New(ctx context.Context, name string, deps worker2.Dep, pool *worker2.Engine, quitWhenDone bool) *Model { return &Model{ name: name, deps: deps, @@ -37,10 +43,10 @@ func New(ctx context.Context, name string, deps *worker.WaitGroup, pool *worker. type Model struct { name string - deps *worker.WaitGroup + deps worker2.Dep start time.Time cancel func() - pool *worker.Pool + pool *worker2.Engine log xtea.LogModel quitWhenDone bool UpdateMessage @@ -63,36 +69,47 @@ func (m *Model) doUpdateMsgTicker() tea.Cmd { func (m *Model) updateMsg(final bool) UpdateMessage { if !final { - final = m.deps.IsDone() - } - - s := m.deps.TransitiveCount() - return UpdateMessage{ - stats: s, - workers: m.pool.Workers, - final: final, + select { + case <-m.deps.Wait(): + final = true + default: + final = false + } } -} -func printJobsWaitStack(jobs []*worker.Job, d int) []string { - prefix := strings.Repeat(" ", d+1) + var workers []workerEntry + for _, w := range m.pool.GetWorkers() { + exec := w.Execution() + if exec == nil { + continue + } - strs := make([]string, 0) - for _, j := range jobs { - if j.IsDone() { + if _, ok := exec.Dep.(*worker2.Group); ok { continue } - strs = append(strs, fmt.Sprintf("%v- %v (%v)", prefix, j.Name, j.State.String())) + var duration time.Duration + if !exec.StartedAt.IsZero() { + duration = time.Since(exec.StartedAt) + } - deps := j.Deps.Jobs() - if len(deps) > 0 { - strs = append(strs, prefix+fmt.Sprintf(" deps: (%v)", len(deps))) - strs = append(strs, printJobsWaitStack(deps, d+1)...) + if duration < 200*time.Millisecond { + continue } + + workers = append(workers, workerEntry{ + status: w.GetStatus(), + duration: duration, + exec: exec, + }) } - return strs + s := worker2.CollectStats(m.deps) + return UpdateMessage{ + stats: s, + workers: workers, + final: final, + } } func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { @@ -107,25 +124,6 @@ func (m *Model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { case tea.KeyBreak: m.cancel() return m, nil - case tea.KeyRunes: - switch msg.String() { - case "p": - jobs := m.pool.Jobs() - - strs := make([]string, 0) - strs = append(strs, "Unfinished jobs:") - strs = append(strs, printJobsWaitStack(jobs, 0)...) - strs = append(strs, "Suspended jobs:") - strs = append(strs, ads.Map( - ads.Filter(jobs, func(job *worker.Job) bool { - return job.State == worker.StateSuspended - }), func(t *worker.Job) string { - return t.Name - }, - )...) - - return m, tea.Println(strings.Join(strs, "\n")) - } } case UpdateMessage: m.UpdateMessage = msg @@ -150,9 +148,9 @@ func (m *Model) View() string { start := xtime.RoundDuration(time.Since(m.start), 1).String() if m.final { - count := fmt.Sprint(m.stats.Done) - if m.stats.Done != m.stats.All { - count = fmt.Sprintf("%v/%v", m.stats.Done, m.stats.All) + count := fmt.Sprint(m.stats.Completed) + if m.stats.Completed != m.stats.All { + count = fmt.Sprintf("%v/%v", m.stats.Completed, m.stats.All) } extra := "" if m.stats.Failed > 0 || m.stats.Skipped > 0 { @@ -162,7 +160,7 @@ func (m *Model) View() string { } var s strings.Builder - s.WriteString(fmt.Sprintf("%v: %v/%v %v", m.name, m.stats.Done, m.stats.All, start)) + s.WriteString(fmt.Sprintf("%v: %v/%v %v", m.name, m.stats.Completed, m.stats.All, start)) if m.stats.Suspended > 0 { s.WriteString(fmt.Sprintf(" (%v suspended)", m.stats.Suspended)) } @@ -172,17 +170,14 @@ func (m *Model) View() string { } for _, w := range m.workers { - runtime := "" - if j := w.CurrentJob; j != nil { - runtime = fmt.Sprintf("=> [%5s]", xtime.FormatFixedWidthDuration(time.Since(j.TimeStart))) - } + runtime := fmt.Sprintf("=> [%5s]", xtime.FormatFixedWidthDuration(w.duration)) - status := w.GetStatus().String(log.Renderer()) - if status == "" { - status = styleFaint.Render("=|") + statusStr := w.status.String(log.Renderer()) + if statusStr == "" { + statusStr = styleFaint.Render("=> Thinking...") } - s.WriteString(fmt.Sprintf("%v %v\n", styleWorkerStart.Render(runtime), status)) + s.WriteString(fmt.Sprintf("%v %v\n", styleWorkerStart.Render(runtime), statusStr)) } return s.String() diff --git a/worker2/poolwait/http.go b/worker2/poolwait/http.go new file mode 100644 index 00000000..26a8ff6b --- /dev/null +++ b/worker2/poolwait/http.go @@ -0,0 +1,53 @@ +package poolwait + +import ( + "fmt" + "github.com/hephbuild/heph/log/log" + "github.com/hephbuild/heph/worker2" + "net" + "net/http" + "time" +) + +func Server(dep worker2.Dep) (func() error, error) { + l, err := net.Listen("tcp", ":0") + if err != nil { + return nil, err + } + + log.Infof("poolwait server listening at http://%v", l.Addr().String()) + + go func() { + doneCh := make(chan struct{}) + defer close(doneCh) + go func() { + for { + select { + case <-doneCh: + return + case <-time.After(time.Second): + log.Infof("poolwait server listening at http://%v", l.Addr().String()) + } + + } + }() + + err := http.Serve(l, http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { + dep.DeepDo(func(dep worker2.Dep) { + if dep.GetState().IsFinal() { + return + } + fmt.Fprintf(rw, "# %v %v\n", dep.GetName(), dep.GetState()) + debugString := dep.GetExecutionDebugString() + if debugString != "" { + fmt.Fprintf(rw, "%v\n\n", debugString) + } + }) + })) + if err != nil { + log.Error("polwaitserver", err) + } + }() + + return l.Close, nil +} diff --git a/worker/poolwait/log.go b/worker2/poolwait/log.go similarity index 61% rename from worker/poolwait/log.go rename to worker2/poolwait/log.go index 12509a26..961bcc27 100644 --- a/worker/poolwait/log.go +++ b/worker2/poolwait/log.go @@ -4,16 +4,27 @@ import ( "fmt" "github.com/hephbuild/heph/log/log" "github.com/hephbuild/heph/utils/xtime" - "github.com/hephbuild/heph/worker" + "github.com/hephbuild/heph/worker2" "os" "strings" "time" ) -func logUI(name string, deps *worker.WaitGroup, pool *worker.Pool) error { +func printWhatItsWaitingOn(dep worker2.Dep, indent string) { + fmt.Println(indent, dep.GetName(), dep.GetState().String(), ":", len(dep.GetNode().Dependencies.Values()), "deps") + for _, d := range dep.GetNode().Dependencies.Values() { + if d.GetState().IsFinal() { + return + } + + printWhatItsWaitingOn(d, " "+indent) + } +} + +func logUI(name string, deps worker2.Dep, pool *worker2.Engine, interval time.Duration) error { start := time.Now() printProgress := func() { - s := deps.TransitiveCount() + s := worker2.CollectStats(deps) extra := "" if s.Failed > 0 || s.Skipped > 0 || s.Suspended > 0 { @@ -31,17 +42,17 @@ func logUI(name string, deps *worker.WaitGroup, pool *worker.Pool) error { extra += ")" } - log.Infof("Progress %v: %v/%v %v%v", name, s.Done, s.All, xtime.RoundDuration(time.Since(start), 1).String(), extra) + log.Infof("Progress %v: %v/%v %v%v", name, s.Completed, s.All, xtime.RoundDuration(time.Since(start), 1).String(), extra) } printWorkersStatus := func() { statusm := map[string]struct{}{} - for _, job := range pool.Jobs() { - if job.State != worker.StateSuspended { + for _, job := range pool.GetWorkers() { + if job.Execution().State != worker2.ExecStateSuspended { continue } - duration := time.Since(job.TimeStart) + duration := time.Since(job.Execution().StartedAt) status := job.GetStatus().String(log.Renderer()) if status == "" { @@ -60,13 +71,13 @@ func logUI(name string, deps *worker.WaitGroup, pool *worker.Pool) error { if len(statusm) > 0 { fmt.Fprintf(os.Stderr, "===\n") } - for _, w := range pool.Workers { - j := w.CurrentJob + for _, w := range pool.GetWorkers() { + j := w.Execution() if j == nil { continue } - duration := time.Since(j.TimeStart) + duration := time.Since(j.StartedAt) if duration < 5*time.Second { // Skip printing short jobs @@ -84,7 +95,12 @@ func logUI(name string, deps *worker.WaitGroup, pool *worker.Pool) error { } } - t := time.NewTicker(time.Second) + if interval == 0 { + <-deps.Wait() + return nil + } + + t := time.NewTicker(interval) defer t.Stop() c := 1 @@ -95,10 +111,11 @@ func logUI(name string, deps *worker.WaitGroup, pool *worker.Pool) error { if c >= 5 { c = 1 printWorkersStatus() + //printWhatItsWaitingOn(deps, "") } c++ continue - case <-deps.Done(): + case <-deps.Wait(): // will break } diff --git a/worker2/poolwait/tui.go b/worker2/poolwait/tui.go new file mode 100644 index 00000000..fc6601ac --- /dev/null +++ b/worker2/poolwait/tui.go @@ -0,0 +1,27 @@ +package poolwait + +import ( + "context" + "github.com/hephbuild/heph/utils/xtea" + "github.com/hephbuild/heph/worker2" + "github.com/hephbuild/heph/worker2/poolui" + "time" +) + +func termUI(ctx context.Context, name string, deps worker2.Dep, pool *worker2.Engine) error { + if !xtea.SingleflightTry() { + return logUI(name, deps, pool, time.Second) + } + + defer xtea.SingleflightDone() + + m := poolui.New(ctx, name, deps, pool, true) + defer m.Clean() + + err := xtea.RunModel(m) + if err != nil { + return err + } + + return nil +} diff --git a/worker2/poolwait/wait.go b/worker2/poolwait/wait.go new file mode 100644 index 00000000..81f9c67c --- /dev/null +++ b/worker2/poolwait/wait.go @@ -0,0 +1,51 @@ +package poolwait + +import ( + "context" + "fmt" + "github.com/hephbuild/heph/log/log" + "github.com/hephbuild/heph/utils/xtea" + "github.com/hephbuild/heph/worker2" + "os" + "strconv" + "time" +) + +var debug bool + +func init() { + debug, _ = strconv.ParseBool(os.Getenv("HEPH_DEBUG_POOLWAIT")) +} + +func Wait(ctx context.Context, name string, pool *worker2.Engine, deps worker2.Dep, plain bool, interval time.Duration) error { + pool.Schedule(deps) + + if debug { + stopServer, err := Server(deps) + if err != nil { + return err + } + defer stopServer() + } + + useTUI := xtea.IsTerm() && !plain + + log.Tracef("WaitPool %v", name) + defer func() { + log.Tracef("WaitPool %v DONE", name) + }() + + if useTUI { + err := termUI(ctx, name, deps, pool) + if err != nil { + return fmt.Errorf("poolui: %w", err) + } + } else { + err := logUI(name, deps, pool, interval) + if err != nil { + return fmt.Errorf("logpoolui: %w", err) + } + } + + return deps.GetErr() +} diff --git a/worker2/scheduler.go b/worker2/scheduler.go new file mode 100644 index 00000000..a54b7eae --- /dev/null +++ b/worker2/scheduler.go @@ -0,0 +1,150 @@ +package worker2 + +import ( + "fmt" + "maps" + "sync" + "time" +) + +type Scheduler interface { + Schedule(Dep, InStore) error + Done(Dep) +} + +type UnlimitedScheduler struct{} + +func (ls UnlimitedScheduler) Schedule(d Dep, ins InStore) error { + return nil +} + +func (ls UnlimitedScheduler) Done(d Dep) {} + +func NewLimitScheduler(limit int) *LimitScheduler { + return &LimitScheduler{ + ch: make(chan struct{}, limit), + } +} + +type LimitScheduler struct { + ch chan struct{} +} + +func (ls *LimitScheduler) Schedule(d Dep, ins InStore) error { + select { + case <-d.GetCtx().Done(): + return d.GetCtx().Err() + case ls.ch <- struct{}{}: + return nil + } +} + +func (ls *LimitScheduler) Done(d Dep) { + <-ls.ch +} + +func NewResourceScheduler(limits map[string]float64, def map[string]float64) *ResourceScheduler { + inuse := map[string]float64{} + for k := range limits { + inuse[k] = 0 + } + + return &ResourceScheduler{ + signal: make(chan struct{}, 1), + limits: limits, + inuse: inuse, + sessions: map[Dep]map[string]float64{}, + def: def, + } +} + +type ResourceScheduler struct { + m sync.Mutex + signal chan struct{} + limits map[string]float64 + inuse map[string]float64 + sessions map[Dep]map[string]float64 + def map[string]float64 +} + +func (ls *ResourceScheduler) next() { + select { + case ls.signal <- struct{}{}: + default: + } +} + +func (ls *ResourceScheduler) trySchedule(d Dep, request map[string]float64) bool { + ls.m.Lock() + defer ls.m.Unlock() + + for k, v := range request { + if ls.inuse[k]+v > ls.limits[k] { + return false + } + } + + for k, v := range request { + ls.inuse[k] += v + } + + ls.sessions[d] = maps.Clone(request) + + return true +} + +func (ls *ResourceScheduler) Schedule(d Dep, ins InStore) error { + request := d.GetRequest() + + if request == nil { + request = ls.def + } + + if len(request) == 0 { + return nil + } + + for k, rv := range request { + lv, ok := ls.limits[k] + if !ok { + return fmt.Errorf("unknown resource: %v", k) + } + + if rv > lv { + return fmt.Errorf("requesting more resource than available, request %v got %v", rv, lv) + } + } + + // immediately try to schedule + retry := time.After(0) + + for { + select { + case <-d.GetCtx().Done(): + return d.GetCtx().Err() + case <-retry: + case <-ls.signal: + } + + success := ls.trySchedule(d, request) + if success { + return nil + } + retry = time.After(100 * time.Millisecond) + } +} + +func (ls *ResourceScheduler) Done(d Dep) { + ls.m.Lock() + defer ls.m.Unlock() + + s := ls.sessions[d] + + for k, v := range s { + ls.inuse[k] -= v + } + + delete(ls.sessions, d) + + ls.next() +} diff --git a/worker2/scheduler_test.go b/worker2/scheduler_test.go new file mode 100644 index 00000000..7480586c --- /dev/null +++ b/worker2/scheduler_test.go @@ -0,0 +1,83 @@ +package worker2 + +import ( + "github.com/stretchr/testify/require" + "golang.org/x/net/context" + "math/rand" + "sync" + "testing" + "time" +) + +func TestResourceScheduler(t *testing.T) { + s := NewResourceScheduler(map[string]float64{ + "cpu": 1000, + "memory": 4000, + }, nil) + + d1 := NewAction(ActionConfig{ + Requests: map[string]float64{ + "cpu": 100, + }, + }) + err := s.Schedule(d1, nil) + require.NoError(t, err) + + d2 := NewAction(ActionConfig{ + Requests: map[string]float64{ + "cpu": 100, + }, + }) + err = s.Schedule(d2, nil) + require.NoError(t, err) + + ctx, _ := context.WithTimeout(context.Background(), time.Second) + d3 := NewAction(ActionConfig{ + Ctx: ctx, + Requests: map[string]float64{ + "cpu": 1000, + }, + }) + err = s.Schedule(d3, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + + s.Done(d1) + s.Done(d2) + + d4 := NewAction(ActionConfig{ + Requests: map[string]float64{ + "cpu": 1000, + }, + }) + err = s.Schedule(d4, nil) + require.NoError(t, err) +} + +func TestStressResourceScheduler(t *testing.T) { + s := NewResourceScheduler(map[string]float64{ + "cpu": 1000, + }, nil) + + var wg sync.WaitGroup + + for i := 0; i < 10000; i++ { + wg.Add(1) + go func() { + d := NewAction(ActionConfig{ + Requests: map[string]float64{ + "cpu": float64(rand.Intn(200)), + }, + }) + err := s.Schedule(d, nil) + require.NoError(t, err) + + go func() { + time.Sleep(time.Millisecond) + s.Done(d) + wg.Done() + }() + }() + } + + wg.Wait() +} diff --git a/worker2/store.go b/worker2/store.go new file mode 100644 index 00000000..6c66e955 --- /dev/null +++ b/worker2/store.go @@ -0,0 +1,30 @@ +package worker2 + +type InStore interface { + Get(key string) any +} + +type OutStore interface { + Set(Value) + Get() Value +} + +type inStore struct { + m map[string]any +} + +func (s *inStore) Get(name string) any { + return s.m[name] +} + +type outStore struct { + value Value +} + +func (s *outStore) Set(v Value) { + s.value = v +} + +func (s *outStore) Get() Value { + return s.value +} diff --git a/worker2/suspend.go b/worker2/suspend.go new file mode 100644 index 00000000..4ccd8ddf --- /dev/null +++ b/worker2/suspend.go @@ -0,0 +1,73 @@ +package worker2 + +import "context" + +var executionKey struct{} + +func contextWithExecution(ctx context.Context, e *Execution) context.Context { + return context.WithValue(ctx, executionKey, e) +} + +func executionFromContext(ctx context.Context) *Execution { + e, _ := ctx.Value(executionKey).(*Execution) + return e +} + +func Wait(ctx context.Context, f func()) { + _ = WaitE(ctx, func() error { + f() + return nil + }) +} + +func WaitE(ctx context.Context, f func() error) error { + e := executionFromContext(ctx) + if e == nil { + err := f() + if err != nil { + return err + } + } else { + sb := e.Suspend() + err := f() + ack := sb.Resume() + <-ack + if err != nil { + return err + } + } + return nil +} + +func WaitDep(ctx context.Context, dep Dep) error { + return WaitE(ctx, func() error { + select { + case <-dep.Wait(): + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) +} + +func WaitChan[T any](ctx context.Context, ch <-chan T) error { + return WaitE(ctx, func() error { + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) +} + +func WaitChanE[T error](ctx context.Context, ch <-chan T) error { + return WaitE(ctx, func() error { + select { + case err := <-ch: + return err + case <-ctx.Done(): + return ctx.Err() + } + }) +} diff --git a/worker2/tracker.go b/worker2/tracker.go new file mode 100644 index 00000000..6af48dfb --- /dev/null +++ b/worker2/tracker.go @@ -0,0 +1,46 @@ +package worker2 + +import ( + "github.com/hephbuild/heph/utils/sets" +) + +func NewRunningTracker() *RunningTracker { + return &RunningTracker{ + running: sets.NewIdentitySet[*Execution](0), + group: NewNamedGroup("tracker"), + } +} + +type RunningTracker struct { + running *sets.Set[*Execution, *Execution] + group *Group +} + +func (t *RunningTracker) Get() []*Execution { + return t.running.Slice() +} + +func (t *RunningTracker) Group() *Group { + return t.group +} + +func (t *RunningTracker) Hook() Hook { + if t == nil { + return nil + } + + return func(event Event) { + switch event := event.(type) { + case EventDeclared: + t.group.AddDep(event.Dep) + case EventScheduled: + t.group.AddDep(event.Execution.Dep) + case EventStarted: + t.running.Add(event.Execution) + case EventSkipped: + t.running.Remove(event.Execution) + case EventCompleted: + t.running.Remove(event.Execution) + } + } +} diff --git a/worker2/value.go b/worker2/value.go new file mode 100644 index 00000000..28fd5773 --- /dev/null +++ b/worker2/value.go @@ -0,0 +1,39 @@ +package worker2 + +type Value interface { + Get() (any, error) +} + +func NewValue[T any](v T) MemValue[T] { + return MemValue[T]{V: v} +} + +type MemValue[T any] struct { + V T +} + +func (v MemValue[T]) Get() (any, error) { + return v.V, nil +} + +type MapValue map[string]Value + +func (m MapValue) Get() (any, error) { + out := make(map[string]any, len(m)) + for k, vv := range m { + if vv == nil { + continue + } + v, err := vv.Get() + if err != nil { + return nil, err + } + out[k] = v + } + + return out, nil +} + +func (m MapValue) Set(k string, v Value) { + m[k] = v +} diff --git a/worker2/worker.go b/worker2/worker.go new file mode 100644 index 00000000..e2b86394 --- /dev/null +++ b/worker2/worker.go @@ -0,0 +1,58 @@ +package worker2 + +import ( + "context" + "errors" + "github.com/hephbuild/heph/status" +) + +type Worker struct { + ctx context.Context + status status.Statuser + exec *Execution + queue func() +} + +func (w *Worker) Status(status status.Statuser) { + w.status = status +} + +func (w *Worker) GetStatus() status.Statuser { + s := w.status + if s == nil { + s = status.String("") + } + return s +} + +func (w *Worker) Interactive() bool { + return true +} + +func (w *Worker) Execution() *Execution { + return w.exec +} + +func (w *Worker) Run() { + ctx := contextWithExecution(w.ctx, w.exec) + ctx = status.ContextWithHandler(ctx, w) + err := w.exec.Run(ctx) + w.status = nil + w.exec.scheduler.Done(w.exec.Dep) + + var errSuspend ErrSuspended + if errors.As(err, &errSuspend) { + w.exec.eventsCh <- EventSuspended{Execution: w.exec} + + go func() { + <-errSuspend.Bag.WaitResume() + w.queue() + }() + } else { + w.exec.eventsCh <- EventCompleted{ + Execution: w.exec, + Output: w.exec.outStore.Get(), + Error: err, + } + } +} diff --git a/x/BUILD b/x/BUILD index a00d2457..1d6738eb 100644 --- a/x/BUILD +++ b/x/BUILD @@ -47,7 +47,8 @@ done waitfail = target( name = "wait-fail", run = """ - sleep 5 + echo will fail... + sleep 1 exit 1 """, cache = False, @@ -112,3 +113,22 @@ target( }, }, ) + +reqdeps = [] +for i in range(0, 3): + t = target( + name = "req{}".format(i), + run = "sleep 2", + cache = False, + requests = { + "cpu": heph.num_cpu() + } + ) + reqdeps.append(t) + +target( + name = "req", + run = "echo ran", + deps = reqdeps, + cache = False, +)