Multi-attribute index, another progressbar, fixup with UniqueSource s…

…et to DomainPart when loading, RootDSE is synthesized and imported, performance optimizations

collector/main.go

@@ -1,6 +1,9 @@
 package main
 
 import (
+	"fmt"
+	"os"
+
 	"github.com/lkarlslund/adalanche/modules/integrations/localmachine/collect"
 	"github.com/lkarlslund/adalanche/modules/version"
 	"github.com/mattn/go-colorable"
@@ -31,6 +34,14 @@ func Execute(cmd *cobra.Command, args []string) error {
 		log.Info().Msg("Debug logging enabled")
 	}
 
+	// Ensure the data folder is available
+	if _, err := os.Stat(*datapath); os.IsNotExist(err) {
+		err = os.MkdirAll(*datapath, 0711)
+		if err != nil {
+			return fmt.Errorf("Could not create data folder %v: %v", datapath, err)
+		}
+	}
+
 	return collect.Collect(*datapath)
 }
 

go.mod

@@ -119,10 +119,10 @@ require (
 	go.uber.org/multierr v1.6.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
 	golang.org/x/crypto v0.0.0-20220307211146-efcb8507fb70 // indirect
-	golang.org/x/mod v0.5.0 // indirect
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
-	golang.org/x/tools v0.1.7 // indirect
+	golang.org/x/tools v0.1.10 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/fsnotify.v1 v1.4.7 // indirect
@@ -134,4 +134,8 @@ require (
 	howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
 )
 
-require github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0 // indirect
+require (
+	github.com/dmarkham/enumer v1.5.5 // indirect
+	github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0 // indirect
+	github.com/pascaldekloe/name v1.0.0 // indirect
+)

go.sum

@@ -156,6 +156,8 @@ github.com/dchest/safefile v0.0.0-20151022103144-855e8d98f185/go.mod h1:cFRxtTwT
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91 h1:Izz0+t1Z5nI16/II7vuEo/nHjodOg0p7+OiDpjX5t1E=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dmarkham/enumer v1.5.5 h1:LpOGL3PQTPOM87rgowZEf7Z5EmkgnKqUtS92Vo+vqzs=
+github.com/dmarkham/enumer v1.5.5/go.mod h1:qHwULwuCxYFAFM5KCkpF1U/U0BF5sNQKLccvUzKNY2w=
 github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
 github.com/dop251/goja v0.0.0-20220214123719-b09a6bfa842f h1:ztRywKO1rqqS8li0TDcnwi9AGsqAH0ky9NaND69/Ccc=
 github.com/dop251/goja v0.0.0-20220214123719-b09a6bfa842f/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
@@ -559,6 +561,8 @@ github.com/open2b/scriggo v0.52.2 h1:gGwfp+E/1dwZj/9wVDljyTXHBG6v/7GlqS1H0fHvGUU
 github.com/open2b/scriggo v0.52.2/go.mod h1:BT/Y/AXiydiLxhbfupynMyF0n2L9/qB57jTbu5+fgh4=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U=
+github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/philhofer/fwd v1.1.1 h1:GdGcTjf5RNAxwS4QLsiMzJYj5KEvPJD3Abr261yRQXQ=
@@ -781,6 +785,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1034,6 +1040,8 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/tools v0.1.7 h1:6j8CgantCy3yc8JGBqkDLMKWqZ0RDU2g1HVgacojGWQ=
 golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
+golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
+golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

modules/engine/attributevalue.go

@@ -121,19 +121,19 @@ func (avs AttributeValueSlice) Len() int {
 }
 
 type AttributeValueOne struct {
-	AttributeValue
+	Value AttributeValue
 }
 
 func (avo AttributeValueOne) Len() int {
 	return 1
 }
 
 func (avo AttributeValueOne) Slice() []AttributeValue {
-	return AttributeValueSlice{avo.AttributeValue}
+	return AttributeValueSlice{avo.Value}
 }
 
 func (avo AttributeValueOne) StringSlice() []string {
-	return []string{avo.AttributeValue.String()}
+	return []string{avo.Value.String()}
 }
 
 type AttributeValue interface {

modules/engine/object.go

@@ -125,9 +125,13 @@ func (o *Object) runlock() {
 	}
 }
 
+func (o *Object) Absorb(source *Object) {
+	o.AbsorbEx(source, false)
+}
+
 // Absorbs data and Pwn relationships from another object, sucking the soul out of it
 // The sources empty shell should be discarded afterwards (i.e. not appear in an Objects collection)
-func (o *Object) Absorb(source *Object) {
+func (o *Object) AbsorbEx(source *Object, fast bool) {
 	if o == source {
 		log.Fatal().Msg("Can't absorb myself")
 	}
@@ -140,49 +144,53 @@ func (o *Object) Absorb(source *Object) {
 	defer o.unlock()
 
 	target := o
-	for attr, values := range source.values {
-		var val AttributeValues
-		tval := target.attr(attr)
-		sval := values
-
-		if tval.Len() == 0 {
-			val = sval
-		} else if sval.Len() == 0 {
-			panic(fmt.Sprintf("Attribute %v with ZERO LENGTH data failure", attr.String()))
-		} else if tval.Len() == 1 && sval.Len() == 1 {
-			tvalue := tval.Slice()[0]
-			svalue := sval.Slice()[0]
-
-			if CompareAttributeValues(tvalue, svalue) {
-				val = tval // They're the same, so pick any
+
+	// Fast mode does not merge values, it just relinks the source to the target
+	if !fast {
+		for attr, values := range source.values {
+			var val AttributeValues
+			tval := target.attr(attr)
+			sval := values
+
+			if tval.Len() == 0 {
+				val = sval
+			} else if sval.Len() == 0 {
+				panic(fmt.Sprintf("Attribute %v with ZERO LENGTH data failure", attr.String()))
+			} else if tval.Len() == 1 && sval.Len() == 1 {
+				tvalue := tval.Slice()[0]
+				svalue := sval.Slice()[0]
+
+				if CompareAttributeValues(tvalue, svalue) {
+					val = tval // They're the same, so pick any
+				} else {
+					// They're not the same, join them
+					val = AttributeValueSlice{tvalue, svalue}
+				}
 			} else {
-				// They're not the same, join them
-				val = AttributeValueSlice{tvalue, svalue}
-			}
-		} else {
-			// One or more of them have more than one value, do it the hard way
-			tvalslice := tval.Slice()
-			svalslice := sval.Slice()
-
-			resultingvalues := make([]AttributeValue, tval.Len())
-			copy(resultingvalues, tvalslice)
-
-			for _, svalue := range svalslice {
-				var alreadythere bool
-			compareloop:
-				for _, tvalue := range tvalslice {
-					if CompareAttributeValues(svalue, tvalue) { // Crap!!
-						alreadythere = true
-						break compareloop
+				// One or more of them have more than one value, do it the hard way
+				tvalslice := tval.Slice()
+				svalslice := sval.Slice()
+
+				resultingvalues := make([]AttributeValue, tval.Len())
+				copy(resultingvalues, tvalslice)
+
+				for _, svalue := range svalslice {
+					var alreadythere bool
+				compareloop:
+					for _, tvalue := range tvalslice {
+						if CompareAttributeValues(svalue, tvalue) { // Crap!!
+							alreadythere = true
+							break compareloop
+						}
+					}
+					if !alreadythere {
+						resultingvalues = append(resultingvalues, svalue)
 					}
 				}
-				if !alreadythere {
-					resultingvalues = append(resultingvalues, svalue)
-				}
+				val = AttributeValueSlice(resultingvalues)
 			}
-			val = AttributeValueSlice(resultingvalues)
+			target.set(attr, val)
 		}
-		target.set(attr, val)
 	}
 
 	for pwntarget, methods := range source.CanPwn {
@@ -234,15 +242,15 @@ func (o *Object) Absorb(source *Object) {
 	source.memberofrecursive = nil
 
 	for _, child := range source.children {
-		target.Adopt(child)
+		target.adopt(child)
 	}
 
 	// If the source has a parent, but the target doesn't we assimilate that role (muhahaha)
 	if source.parent != nil {
 		if target.parent == nil {
 			target.ChildOf(source.parent)
 		}
-		source.parent.RemoveChild(source)
+		source.parent.removeChild(source)
 		source.parent = nil
 	}
 
@@ -472,7 +480,7 @@ func (o *Object) OneAttrString(attr Attribute) string {
 		return ""
 	}
 	if ao, ok := a.(AttributeValueOne); ok {
-		return ao.String()
+		return ao.Value.String()
 	}
 	if a.Len() == 1 {
 		log.Warn().Msg("Inefficient attribute storage - multival used for one value ...")
@@ -772,6 +780,8 @@ func (o *Object) setFlex(flexinit ...interface{}) {
 				continue
 			}
 			data = append(data, v)
+		case AttributeValueOne:
+			data = append(data, v.Value)
 		case AttributeValueSlice:
 			for _, value := range v {
 				if ignoreblanks && value.IsZero() {
@@ -1082,13 +1092,24 @@ func (o *Object) Adopt(child *Object) {
 
 	child.lock()
 	if child.parent != nil {
-		child.parent.RemoveChild(child)
+		child.parent.lock()
+		child.parent.removeChild(child)
+		child.parent.unlock()
 	}
 	child.parent = o
 	child.unlock()
 }
 
-func (o *Object) RemoveChild(child *Object) {
+func (o *Object) adopt(child *Object) {
+	o.children = append(o.children, child)
+
+	if child.parent != nil {
+		child.parent.removeChild(child)
+	}
+	child.parent = o
+}
+
+func (o *Object) removeChild(child *Object) {
 	for i, curchild := range o.children {
 		if curchild == child {
 			if len(o.children) == 1 {