More refactoring and better performance at the cost of some more RAM

go.mod

@@ -47,7 +47,6 @@ require (
 	github.com/antchfx/xpath v1.2.1 // indirect
 	github.com/asergeyev/nradix v0.0.0-20220715161825-e451993e425c // indirect
 	github.com/buger/jsonparser v1.1.1 // indirect
-	github.com/cloudflare/buffer v0.0.0-20190408164202-7cab898e1166 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/crewjam/rfc5424 v0.1.0 // indirect
 	github.com/elastic/beats v7.6.2+incompatible // indirect
@@ -58,7 +57,6 @@ require (
 	github.com/go-playground/validator/v10 v10.11.1 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/cabbie v1.0.4 // indirect
 	github.com/google/glazier v0.0.0-20221020175949-24442e3c310c // indirect
@@ -69,7 +67,6 @@ require (
 	github.com/gravwell/gcfg v1.2.9-0.20220128204816-1742bc68c091 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf // indirect
-	github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/compress v1.15.12 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
@@ -113,43 +110,20 @@ require (
 
 require (
 	github.com/DataDog/gostackparse v0.6.0 // indirect
-	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/dchest/safefile v0.0.0-20151022103144-855e8d98f185 // indirect
-	github.com/dlclark/regexp2 v1.7.0 // indirect
-	github.com/dop251/goja v0.0.0-20221025165401-cb5011b539fe // indirect
-	github.com/dop251/goja_nodejs v0.0.0-20221009164102-3aa5028e57f6 // indirect
-	github.com/elastic/go-sysinfo v1.8.1 // indirect
-	github.com/elastic/go-ucfg v0.8.6 // indirect
-	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/goccy/go-json v0.9.11 // indirect
-	github.com/google/aukera v0.0.0-20201117230544-d145c8357fea // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gopacket v1.1.19 // indirect
-	github.com/google/logger v1.1.1 // indirect
-	github.com/google/subcommands v1.2.0 // indirect
 	github.com/gravwell/buffer v0.0.0-20220728204757-23339f4bab66 // indirect
 	github.com/gravwell/ipfix v1.4.5 // indirect
 	github.com/h2non/filetype v1.1.3 // indirect
-	github.com/iamacarpet/go-win64api v0.0.0-20210311141720-fe38760bed28 // indirect
-	github.com/jstemmer/go-junit-report v0.9.1 // indirect
 	github.com/k-sone/ipmigo v0.0.0-20190922011749-b22c7a70e949 // indirect
-	github.com/magefile/mage v1.14.0 // indirect
 	github.com/miekg/dns v1.1.50 // indirect
-	github.com/mitchellh/hashstructure v1.1.0 // indirect
 	github.com/open-networks/go-msgraph v0.3.1 // indirect
 	github.com/open2b/scriggo v0.56.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.5 // indirect
-	github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/tealeg/xlsx v1.0.5 // indirect
-	go.uber.org/atomic v1.7.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
-	go.uber.org/zap v1.23.0 // indirect
 	golang.org/x/mod v0.6.0 // indirect
 	golang.org/x/tools v0.2.0 // indirect
-	gopkg.in/fsnotify.v1 v1.4.7 // indirect
 	gopkg.in/gcfg.v1 v1.2.3 // indirect
-	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
 )

modules/analyze/webservicefuncs.go

@@ -101,7 +101,7 @@ func analysisfuncs(ws *webservice) {
 				w.Write([]byte(err.Error()))
 				return
 			}
-			o, found = ws.Objs.FindByID(uint32(id))
+			o, found = ws.Objs.FindID(engine.ObjectID(id))
 		case "dn", "distinguishedname":
 			o, found = ws.Objs.Find(activedirectory.DistinguishedName, engine.AttributeValueString(vars["id"]))
 		case "sid":
@@ -792,7 +792,7 @@ func analysisfuncs(ws *webservice) {
 		encoder := qjson.NewEncoder(w)
 		encoder.SetIndent("", "  ")
 
-		var children []*engine.Object
+		var children engine.ObjectSlice
 		if idstr == "#" {
 			children = ws.Objs.Root().Children()
 		} else {
@@ -803,7 +803,7 @@ func analysisfuncs(ws *webservice) {
 				return
 			}
 
-			if parent, found := ws.Objs.FindByID(uint32(id)); found {
+			if parent, found := ws.Objs.FindID(engine.ObjectID(id)); found {
 				children = parent.Children()
 			} else {
 				w.WriteHeader(404) // not found
@@ -813,21 +813,22 @@ func analysisfuncs(ws *webservice) {
 		}
 
 		type treeData struct {
-			Label    string `json:"text"`
-			Type     string `json:"type,omitempty"`
-			ID       uint32 `json:"id"`
-			Children bool   `json:"children,omitempty"`
+			Label    string          `json:"text"`
+			Type     string          `json:"type,omitempty"`
+			ID       engine.ObjectID `json:"id"`
+			Children bool            `json:"children,omitempty"`
 		}
 
 		var results []treeData
-		for _, object := range children {
+		children.Iterate(func(object *engine.Object) bool {
 			results = append(results, treeData{
 				ID:       object.ID(),
 				Label:    object.Label(),
 				Type:     object.Type().String(),
-				Children: len(object.Children()) > 0,
+				Children: object.Children().Len() > 0,
 			})
-		}
+			return true
+		})
 
 		err := encoder.Encode(results)
 		if err != nil {

modules/analyze/xgmml.go

@@ -1,6 +1,10 @@
 package analyze
 
-import "encoding/xml"
+import (
+	"encoding/xml"
+
+	"github.com/lkarlslund/adalanche/modules/engine"
+)
 
 // type XGMML struct {
 // 	XMLNAme xml.Name `xml:"graph"`
@@ -35,18 +39,18 @@ type XGMMLGraph struct {
 }
 
 type XGMMLNode struct {
-	XMLName    xml.Name `xml:"node"`
-	Id         uint32   `xml:"id,attr"`
-	Label      string   `xml:"label,attr"`
-	Weight     int      `xml:"weight,attr,omitempty"`
+	XMLName    xml.Name        `xml:"node"`
+	Id         engine.ObjectID `xml:"id,attr"`
+	Label      string          `xml:"label,attr"`
+	Weight     int             `xml:"weight,attr,omitempty"`
 	Attributes []XGMMLAttribute
 }
 
 type XGMMLEdge struct {
-	XMLName    xml.Name `xml:"edge"`
-	Source     uint32   `xml:"source,attr"`
-	Target     uint32   `xml:"target,attr"`
-	Label      string   `xml:"label,attr"`
+	XMLName    xml.Name        `xml:"edge"`
+	Source     engine.ObjectID `xml:"source,attr"`
+	Target     engine.ObjectID `xml:"target,attr"`
+	Label      string          `xml:"label,attr"`
 	Attributes []XGMMLAttribute
 }
 

modules/engine/analyzeobjects.go

@@ -176,7 +176,7 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 				}
 
 				if opts.ExcludeObjects != nil {
-					if _, found := opts.ExcludeObjects.FindByID(target.ID()); found {
+					if _, found := opts.ExcludeObjects.FindID(target.ID()); found {
 						// skip excluded objects
 						// ui.Debug().Msgf("Excluding target %v", pwntarget.DN())
 						return true // continue
@@ -341,7 +341,7 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 		}
 		for node := range implicatedobjectsmap {
 			if _, found := pointedto[node]; !found {
-				if _, found := opts.IncludeObjects.FindByID(node.ID()); opts.PruneIslands || !found {
+				if _, found := opts.IncludeObjects.FindID(node.ID()); opts.PruneIslands || !found {
 					delete(implicatedobjectsmap, node)
 					prunedislands++
 				}
@@ -374,7 +374,7 @@ func AnalyzeObjects(opts AnalyzeObjectsOptions) (pg Graph) {
 	i = 0
 	for object, ri := range implicatedobjectsmap {
 		pg.Nodes[i].Object = object
-		if _, found := opts.IncludeObjects.FindByID(object.ID()); found {
+		if _, found := opts.IncludeObjects.FindID(object.ID()); found {
 			pg.Nodes[i].Target = true
 		}
 		pg.Nodes[i].CanExpand = ri.canexpand

modules/engine/attributes.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/lkarlslund/adalanche/modules/ui"
 )
@@ -14,15 +15,16 @@ type AttributeGetFunc func(o *Object, a Attribute) (v AttributeValues, found boo
 type AttributeSetFunc func(o *Object, a Attribute, v AttributeValues) error
 
 type attributeinfo struct {
-	onset  AttributeSetFunc
-	onget  AttributeGetFunc
-	name   string
-	tags   []string
-	atype  AttributeType
-	single bool // If true, this attribute can not have multiple values
-	unique bool // Doing a Find on this attribute will return multiple results
-	merge  bool // If true, objects can be merged on this attribute
-	hidden bool // If true this does not show up in the list of attributes
+	onset          AttributeSetFunc
+	onget          AttributeGetFunc
+	name           string
+	tags           []string
+	atype          AttributeType
+	mergeSuccesses atomic.Uint64 // number of successfull merges where this attribute was the deciding factor
+	single         bool          // If true, this attribute can not have multiple values
+	unique         bool          // Doing a Find on this attribute will return multiple results
+	merge          bool          // If true, objects can be merged on this attribute
+	hidden         bool          // If true this does not show up in the list of attributes
 }
 
 type AttributeType uint8

modules/engine/dedupvalues.go

@@ -0,0 +1,22 @@
+package engine
+
+import (
+	"sync"
+)
+
+var dedupValues int
+var attributeValueDedupper map[AttributeValue]*AttributeValue
+var dedupLock sync.RWMutex
+
+func DedupValues(enable bool) {
+	if enable {
+		dedupValues++
+	} else {
+		dedupValues--
+	}
+	if dedupValues == 0 {
+		attributeValueDedupper = nil
+	} else if dedupValues > 0 && attributeValueDedupper == nil {
+		attributeValueDedupper = make(map[AttributeValue]*AttributeValue)
+	}
+}

modules/engine/edge.go

@@ -57,6 +57,36 @@ func (eb EdgeBitmap) Set(edge Edge) EdgeBitmap {
 	return eb.set(edge)
 }
 
+func (eb *EdgeBitmap) AtomicSet(edge Edge) {
+	atomic.AddUint64(&EdgePopularity[edge], 1)
+	bits := uint64(1) << (edge % 64)
+	index := int(edge) / 64
+
+	for {
+		oldvalue := atomic.LoadUint64(&eb[index])
+		newvalue := oldvalue | bits
+		if atomic.CompareAndSwapUint64(&eb[index], oldvalue, newvalue) {
+			// We won the race
+			break
+		}
+	}
+}
+
+func (eb *EdgeBitmap) AtomicOr(edges EdgeBitmap) {
+	index := 0
+	for {
+		oldvalue := atomic.LoadUint64(&eb[index])
+		newvalue := oldvalue | edges[index]
+		if atomic.CompareAndSwapUint64(&eb[index], oldvalue, newvalue) {
+			// We won the race
+			index++
+			if index == PMBSIZE {
+				break
+			}
+		}
+	}
+}
+
 func (eb EdgeBitmap) set(edge Edge) EdgeBitmap {
 	newpm := eb
 	bits := uint64(1) << (edge % 64)
@@ -71,6 +101,43 @@ func (eb EdgeBitmap) Clear(edge Edge) EdgeBitmap {
 	return newpm
 }
 
+func (eb *EdgeBitmap) AtomicClear(edge Edge) {
+	atomic.AddUint64(&EdgePopularity[edge], 1)
+	bits := uint64(1) << (edge % 64)
+	index := int(edge) / 64
+
+	for {
+		oldvalue := atomic.LoadUint64(&eb[index])
+		newvalue := oldvalue & ^bits
+		if atomic.CompareAndSwapUint64(&eb[index], oldvalue, newvalue) {
+			// We won the race
+			break
+		}
+	}
+}
+
+func (eb *EdgeBitmap) AtomicAnd(edges EdgeBitmap) {
+	index := 0
+	for {
+		oldvalue := atomic.LoadUint64(&eb[index])
+		newvalue := oldvalue & edges[index]
+		if atomic.CompareAndSwapUint64(&eb[index], oldvalue, newvalue) {
+			// We won the race
+			index++
+			if index == PMBSIZE {
+				break
+			}
+		}
+	}
+}
+
+func (eb EdgeBitmap) Invert() EdgeBitmap {
+	for index := 0; index <= PMBSIZE; index++ {
+		eb[index] = ^eb[index]
+	}
+	return eb
+}
+
 func (eb EdgeBitmap) Intersect(edges EdgeBitmap) EdgeBitmap {
 	var new EdgeBitmap
 	for i := 0; i < PMBSIZE; i++ {