Less allocation in worker dag & fix cache ro

.github/workflows/release.yml

@@ -37,6 +37,11 @@ jobs:
           restore-keys: |
             heph-cache-build-
 
+      - name: Configure golang
+        run: |
+          echo "CGO_ENABLED=0" >> $GITHUB_ENV
+          echo "GOEXPERIMENT=rangefunc" >> $GITHUB_ENV
+
       - name: Bootstrap Build
         run: go build -v -o heph github.com/hephbuild/heph/cmd/heph
 

BUILD

@@ -11,6 +11,9 @@ go_toolchain(
         "linux_amd64",
         "linux_arm64",
     ],
+    env = {
+        "GOEXPERIMENT": "rangefunc",
+    },
 )
 
 node = node_toolchain(
@@ -51,7 +54,7 @@ target(
     cache = False,
 )
 
-go_env_vars = ["GOROOT", "GOPATH", "HOME"]
+go_env_vars = ["GOROOT", "GOPATH", "HOME", "GOEXPERIMENT"]
 
 target(
     name = "test_go",

go.mod

@@ -1,6 +1,6 @@
 module github.com/hephbuild/heph
 
-go 1.21
+go 1.22
 
 replace github.com/spf13/cobra v1.7.0 => github.com/raphaelvigee/cobra v0.0.0-20221020122344-217ca52feee0
 

lcache/artifacts_gen.go

@@ -194,6 +194,8 @@ func (e *LocalCacheState) ScheduleGenArtifacts(ctx context.Context, gtarget grap
 
 	dir := dirp.Abs()
 
+	xfs.MakeDirsReadWrite(dir)
+
 	err = os.RemoveAll(dir)
 	if err != nil {
 		return nil, err

lcache/lcache.go

@@ -407,7 +407,12 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 		Outputs []string
 		CacheRW bool
 	}
-	version := 1
+
+	expectedMeta := OutDirMeta{
+		Version: 1,
+		Outputs: outputs,
+		CacheRW: e.CacheRW,
+	}
 
 	shouldExpand := false
 	shouldCleanExpand := false
@@ -422,11 +427,13 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 		var currentMeta OutDirMeta
 		currentMeta.CacheRW = true // Legacy behavior
 		_ = json.Unmarshal(b, &currentMeta)
-		if currentMeta.Version != version || !ads.ContainsAll(currentMeta.Outputs, outputs) {
+		if currentMeta.Version != expectedMeta.Version || !ads.ContainsAll(currentMeta.Outputs, expectedMeta.Outputs) {
 			shouldExpand = true
-		} else if currentMeta.Version != version {
+		}
+
+		if currentMeta.Version != expectedMeta.Version {
 			shouldCleanExpand = true
-		} else if currentMeta.CacheRW != e.CacheRW {
+		} else if currentMeta.CacheRW != expectedMeta.CacheRW {
 			shouldCleanExpand = true
 		}
 	}
@@ -493,10 +500,7 @@ func (e *LocalCacheState) Expand(ctx context.Context, ttarget graph.Targeter, ou
 			_ = r.Close()
 		}
 
-		b, err := json.Marshal(OutDirMeta{
-			Version: version,
-			Outputs: outputs,
-		})
+		b, err := json.Marshal(expectedMeta)
 		if err != nil {
 			return xfs.Path{}, err
 		}

worker2/dag.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/hephbuild/heph/utils/ads"
 	"github.com/hephbuild/heph/utils/sets"
+	"iter"
 	"strings"
 	"sync"
 )
@@ -17,31 +18,35 @@ type nodesTransitive[T any] struct {
 	transitiveReverse bool
 }
 
-func (d *nodesTransitive[T]) Add(dep *Node[T]) {
-	d.m.Lock()
-	defer d.m.Unlock()
+func (d *nodesTransitive[T]) Add(dep *Node[T]) bool {
+	d.m.RLock()
+	defer d.m.RUnlock()
 
-	d.nodes.Add(dep)
-	d.transitiveDirty = true
+	if d.nodes.Add(dep) {
+		d.transitiveDirty = true
+		return true
+	}
+
+	return false
 }
 
 func (d *nodesTransitive[T]) MarkTransitiveDirty() {
-	d.m.Lock()
-	defer d.m.Unlock()
+	d.m.RLock()
+	defer d.m.RUnlock()
 
 	d.transitiveDirty = true
 }
 
 func (d *nodesTransitive[T]) MarkTransitiveInvalid() {
-	d.m.Lock()
-	defer d.m.Unlock()
+	d.m.RLock()
+	defer d.m.RUnlock()
 
 	d.transitiveNodes = nil
 }
 
 func (d *nodesTransitive[T]) Remove(dep *Node[T]) {
-	d.m.Lock()
-	defer d.m.Unlock()
+	d.m.RLock()
+	defer d.m.RUnlock()
 
 	d.nodes.Remove(dep)
 	d.transitiveNodes = nil
@@ -75,16 +80,20 @@ func (d *nodesTransitive[T]) TransitiveSet() *sets.Set[*Node[T], *Node[T]] {
 	return d.transitiveNodes
 }
 
-func (d *nodesTransitive[T]) TransitiveValues() []T {
-	return ads.Map(d.TransitiveSet().Slice(), func(t *Node[T]) T {
-		return t.V
-	})
+func (d *nodesTransitive[T]) TransitiveValues() iter.Seq2[int, T] {
+	return func(yield func(int, T) bool) {
+		for i, node := range d.TransitiveSet().Slice() {
+			yield(i, node.V)
+		}
+	}
 }
 
-func (d *nodesTransitive[T]) Values() []T {
-	return ads.Map(d.Set().Slice(), func(t *Node[T]) T {
-		return t.V
-	})
+func (d *nodesTransitive[T]) Values() iter.Seq2[int, T] {
+	return func(yield func(int, T) bool) {
+		for i, node := range d.Set().Slice() {
+			yield(i, node.V)
+		}
+	}
 }
 
 func (d *nodesTransitive[T]) computeTransitive(full bool) *sets.Set[*Node[T], *Node[T]] {
@@ -161,15 +170,13 @@ func (d *Node[T]) addDependency(dep *Node[T]) {
 			panic("cycle")
 		}
 
-		d.Dependencies.Add(dep)
-
-		for _, dependee := range d.Dependees.TransitiveSet().Slice() {
-			dependee.Dependencies.MarkTransitiveDirty()
+		if d.Dependencies.Add(dep) {
+			for _, dependee := range d.Dependees.TransitiveSet().Slice() {
+				dependee.Dependencies.MarkTransitiveDirty()
+			}
 		}
 
-		if !dep.Dependees.Has(d) {
-			dep.Dependees.Add(d)
-
+		if dep.Dependees.Add(d) {
 			for _, dep := range dep.Dependencies.TransitiveSet().Slice() {
 				dep.Dependees.MarkTransitiveDirty()
 			}

worker2/dep_utils.go

@@ -1,14 +1,51 @@
 package worker2
 
-import "sync/atomic"
+import (
+	"github.com/hephbuild/heph/utils/xsync"
+	"golang.org/x/exp/maps"
+	"sync/atomic"
+)
 
-func deepDo(a Dep, f func(Dep)) {
+func deepDoPrecomputed(a Dep, f func(Dep)) {
 	f(a)
 	for _, dep := range a.GetNode().Dependencies.TransitiveValues() {
 		f(dep)
 	}
 }
 
+var deepDoMapPool = xsync.Pool[map[Dep]struct{}]{New: func() map[Dep]struct{} {
+	return map[Dep]struct{}{}
+}}
+
+func deepDo(a Dep, f func(Dep)) {
+	if false {
+		// This approach sounds good on paper, but in reality very CPU intensive since it requires
+		// read & write to the deps set at every change, at every level...
+		deepDoPrecomputed(a, f)
+	} else {
+		deepDoRecursive(a, f)
+	}
+}
+
+func deepDoRecursive(a Dep, f func(Dep)) {
+	m := deepDoMapPool.Get()
+	maps.Clear(m)
+	defer deepDoMapPool.Put(m)
+	deepDoRecursiveInner(a, f, m)
+}
+
+func deepDoRecursiveInner(a Dep, f func(Dep), m map[Dep]struct{}) {
+	if _, ok := m[a]; ok {
+		return
+	}
+	m[a] = struct{}{}
+
+	f(a)
+	for _, dep := range a.GetNode().Dependencies.Values() {
+		deepDoRecursiveInner(dep, f, m)
+	}
+}
+
 type Stats struct {
 	All       uint64
 	Completed uint64

worker2/poolwait/log.go

@@ -11,7 +11,7 @@ import (
 )
 
 func printWhatItsWaitingOn(dep worker2.Dep, indent string) {
-	fmt.Println(indent, dep.GetName(), dep.GetState().String(), ":", len(dep.GetNode().Dependencies.Values()), "deps")
+	fmt.Println(indent, dep.GetName(), dep.GetState().String(), ":", dep.GetNode().Dependencies.Set().Len(), "deps")
 	for _, d := range dep.GetNode().Dependencies.Values() {
 		if d.GetState().IsFinal() {
 			return