Skip to content

Commit

Permalink
Switched to upgraded deduplication package
Browse files Browse the repository at this point in the history
  • Loading branch information
@lkarlslund
lkarlslund committed May 30, 2022
1 parent ad64eec commit c1aeb3d
Show file tree
Hide file tree
Showing 7 changed files with 100 additions and 162 deletions.
9 changes: 2 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/lkarlslund/go-win64api v0.0.0-20211005130710-d4f2d07ed091
github.com/lkarlslund/ldap/v3 v3.2.4-0.20210621153959-85555023df29
github.com/lkarlslund/stringdedup v0.2.1
github.com/lkarlslund/stringdedup v0.5.0
github.com/lkarlslund/time-timespan v0.0.0-20210712111050-6e7c565fa001
github.com/mailru/easyjson v0.7.7
github.com/mattn/go-colorable v0.1.12
Expand All @@ -34,9 +34,6 @@ require (
golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
golang.org/x/text v0.3.7
)

require (
github.com/Azure/go-ntlmssp v0.0.0-20211209120228-48547f28849e // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/absfs/absfs v0.0.0-20200602175035-e49edc9fef15 // indirect
Expand Down Expand Up @@ -132,9 +129,7 @@ require (
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect
)

require (
github.com/aloknerurkar/gpool v0.0.0-20220411083022-1c09ad956d39 // indirect
github.com/dmarkham/enumer v1.5.5 // indirect
github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0 // indirect
github.com/pascaldekloe/name v1.0.0 // indirect
Expand Down
3 changes: 3 additions & 0 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74 h1:Kk6a4nehpJ3UuJRqlA3JxYxBZEqCeOmATOvrbT4p9RA=
github.com/alexbrainman/sspi v0.0.0-20210105120005-909beea2cc74/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4=
github.com/aloknerurkar/gpool v0.0.0-20220411083022-1c09ad956d39 h1:sd3phCayLVcPxGwLMoZV+C/LNQkiFsMCljIdCRw64Cw=
github.com/aloknerurkar/gpool v0.0.0-20220411083022-1c09ad956d39/go.mod h1:ty2tNd552qlJEqGbx9wpDr7ImFV2KHVtJ+19N/V/r8E=
github.com/amidaware/taskmaster v0.0.0-20220111015025-c9cd178bbbf2 h1:1K03qwtvgdJRXJr0nE1qvzFPOmbWHBnnnbeblU7+Bg8=
github.com/amidaware/taskmaster v0.0.0-20220111015025-c9cd178bbbf2/go.mod h1:5UVBogOiPFWC2F6fKT/Kb9qD4NCsp2y+dCj+pvXnDQE=
github.com/antchfx/xmlquery v1.3.7 h1:0hc7OU2rFIu8MMKT5kknruaTKsoCybkUaUFMnB1LOO4=
Expand Down Expand Up @@ -486,6 +488,7 @@ github.com/lkarlslund/ldap/v3 v3.2.4-0.20210621153959-85555023df29 h1:4fuGT7A0Tb
github.com/lkarlslund/ldap/v3 v3.2.4-0.20210621153959-85555023df29/go.mod h1:Q/0KZGCdyOiOp6oN+HFifXw9rwBwVB6NkNL4tXXDF70=
github.com/lkarlslund/stringdedup v0.2.1 h1:MFDyr/uY2a8A+gBkCcMlA9ePnGnTzXZS5DV0+mnOSZM=
github.com/lkarlslund/stringdedup v0.2.1/go.mod h1:746ZZPjQHDoKM6mlwEKDiU9ub5CFDx1KL8xzQm/lTSA=
github.com/lkarlslund/stringdedup v0.5.0/go.mod h1:UIhe+EbxkghJObY4d4J8WLp9Xe31tzhcbqRJHxAza/U=
github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0 h1:v56FZeDrnIamOapT2KOO8Vev1a4QdAQ9QKLplnsKQ2M=
github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0/go.mod h1:iYoXC1os3S1gI7/SslO+cTU7dCl/bYhBqmQyVeAM2bw=
github.com/lkarlslund/time-timespan v0.0.0-20210712111050-6e7c565fa001 h1:S9Xs8FSeBXkzHyvXHcYQKFqwr63T+VL6C9I+4q+0z4E=
Expand Down
135 changes: 17 additions & 118 deletions modules/analyze/cli.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"time"

"github.com/lkarlslund/adalanche/modules/cli"
"github.com/lkarlslund/adalanche/modules/dedup"
"github.com/lkarlslund/adalanche/modules/engine"
"github.com/rs/zerolog/log"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -45,126 +46,24 @@ func Execute(cmd *cobra.Command, args []string) error {
// After all this loading and merging, it's time to do release unused RAM
debug.FreeOSMemory()

/*
switch command {
case "schemagraph":
gv := gographviz.NewEscape()
gv.SetName("schema")
gv.SetDir(true)
// gv.AddSubGraph("schema", "attributes", nil)
// gv.AddSubGraph("schema", "classes", nil)
// gv.AddSubGraph("schema", "rights", nil)
log.Info().Msg("Exporting schema graph in Graphviz format ...")
output, err := os.Create("schemagraph.dot")
if err != nil {
log.Fatal().Msgf("Error opening output file: %v", err)
}
for _, object := range objs.Slice() {
switch object.Type() {
case engine.ObjectTypeAttributeSchema:
// gv.AddNode("schema", object.IDString(), map[string]string{"label": object.OneAttrString(LDAPDisplayName)})
// // Part of attribute set?
// if as := object.OneAttr(AttributeSecurityGUID); as != nil {
// if rg, found := engine.AllObjects.Find(RightsGUID, as); found {
// // _ = rg
// gv.AddEdge(object.IDString(), rg.IDString(), true, map[string]string{"label": "Part of"})
// }
// }
//
case engine.ObjectTypeClassSchema:
gv.AddNode("schema", object.IDString(), map[string]string{"label": object.OneAttrString(engine.LDAPDisplayName)})
// Possible superiors
for _, psup := range object.Attr(engine.PossSuperiors).Slice() {
if sup, found := objs.Find(engine.LDAPDisplayName, psup); found {
// _ = sup
gv.AddEdge(sup.IDString(), object.IDString(), true, map[string]string{"label": "Superior"})
}
}
// // Must contain
// for _, pcontain := range object.Attr(SystemMustContain).Slice() {
// if contain, found := engine.AllObjects.Find(LDAPDisplayName, pcontain); found {
// // _ = contain
// gv.AddEdge(object.IDString(), contain.IDString(), true, map[string]string{"label": "Must"})
// }
// }
// // May contain
// for _, pcontain := range object.Attr(SystemMayContain).Slice() {
// if contain, found := engine.AllObjects.Find(LDAPDisplayName, pcontain); found {
// // _ = contain
// gv.AddEdge(object.IDString(), contain.IDString(), true, map[string]string{"label": "May"})
// }
// }
case engine.ObjectTypeControlAccessRight:
gv.AddNode("schema", object.IDString(), map[string]string{"label": object.OneAttrString(engine.DisplayName)})
}
}
output.WriteString(gv.String())
output.Close()
log.Info().Msg("Done")
case "exportobjectsdebug":
log.Info().Msg("Finding most valuable assets ...")
output, err := os.Create("debug.txt")
if err != nil {
log.Fatal().Msgf("Error opening output file: %v", err)
}
for _, object := range objs.Slice() {
fmt.Fprintf(output, "Object:\n%v\n\n-----------------------------\n", object)
}
output.Close()
log.Info().Msg("Done")
case "export":
log.Info().Msg("Finding most valuable assets ...")
q, err := ldapquery.ParseQueryStrict(*analyzequery, objs)
if err != nil {
log.Fatal().Msgf("Error parsing LDAP query: %v", err)
}
includeobjects := objs.Filter(func(o *engine.Object) bool {
return q.Evaluate(o)
})
opts := engine.NewAnalyzeObjectsOptions()
opts.IncludeObjects = includeobjects
opts.Reverse = *exportinverted
resultgraph := engine.AnalyzeObjects(opts)
switch *exporttype {
case "graphviz":
err = ExportGraphViz(resultgraph, "adalanche-"+*domain+".dot")
case "cytoscapejs":
err = ExportCytoscapeJS(resultgraph, "adalanche-cytoscape-js-"+*domain+".json")
default:
log.Error().Msg("Unknown export format")
showUsage()
}
if err != nil {
log.Fatal().Msgf("Problem exporting graph: %v", err)
}
log.Info().Msg("Done")
case "analyze", "dump-analyze":
default:
log.Error().Msgf("Unknown command %v", flag.Arg(0))
showUsage()
}
*/

log.Info().Msgf("Processing done in %v", time.Since(starttime))

dedupStats := dedup.D.Statistics()

log.Debug().Msgf("Deduplicator stats: %v items added using %v bytes in memory", dedupStats.ItemsAdded, dedupStats.BytesInMemory)
log.Debug().Msgf("Deduplicator stats: %v items not allocated saving %v bytes of memory", dedupStats.ItemsSaved, dedupStats.BytesSaved)
log.Debug().Msgf("Deduplicator stats: %v items removed (memory stats unavailable)", dedupStats.ItemsRemoved)
log.Debug().Msgf("Deduplicator stats: %v collisions detected (first at %v objects)", dedupStats.Collisions, dedupStats.FirstCollisionDetected)
log.Debug().Msgf("Deduplicator stats: %v keepalive objects added", dedupStats.KeepAliveItemsAdded)
log.Debug().Msgf("Deduplicator stats: %v keepalive objects removed", dedupStats.KeepAliveItemsRemoved)

// Try to recover some memory
dedup.D.Flush()
objs.DropIndexes()

runtime.GC()
debug.FreeOSMemory()

err = WebService.Start(*bind, objs, *localhtml)
if err != nil {
return err
Expand Down
16 changes: 16 additions & 0 deletions modules/dedup/dedup.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package dedup

import (
"github.com/OneOfOne/xxhash"
"github.com/lkarlslund/stringdedup"
)

var D = stringdedup.New(func(in []byte) uint32 {
return xxhash.Checksum32(in)
})

func init() {
// dedup.DontValidateResults = true
D.CalculateStatistics = true
// D.KeepAlive = 600 * time.Second
}
2 changes: 1 addition & 1 deletion modules/engine/attributevalue.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ func (as AttributeValueString) IsZero() bool {
return len(as) == 0
}

type AttributeValueBlob []byte
type AttributeValueBlob string

func (ab AttributeValueBlob) String() string {
return fmt.Sprintf("% x", []byte(ab))
Expand Down
91 changes: 58 additions & 33 deletions modules/engine/object.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/gofrs/uuid"
"github.com/icza/gox/stringsx"
jsoniter "github.com/json-iterator/go"
"github.com/lkarlslund/adalanche/modules/dedup"
"github.com/lkarlslund/adalanche/modules/util"
"github.com/lkarlslund/adalanche/modules/windowssecurity"
"github.com/lkarlslund/stringdedup"
Expand Down Expand Up @@ -483,7 +484,7 @@ func (o *Object) OneAttrString(attr Attribute) string {
return ao.Value.String()
}
if a.Len() == 1 {
log.Warn().Msg("Inefficient attribute storage - multival used for one value ...")
log.Warn().Msgf("Inefficient attribute storage for %v - multival used for one value ...", attr.String())
return a.Slice()[0].String()
}
log.Error().Msgf("Attribute %v lookup for ONE value, but contains %v (%v)", attr.String(), a.Len(), strings.Join(a.StringSlice(), ", "))
Expand Down Expand Up @@ -675,7 +676,11 @@ func (o *Object) SetValues(a Attribute, values ...AttributeValue) {
if len(values) == 0 {
panic(fmt.Sprintf("tried to set attribute %v to NO values", a.String()))
}
o.Set(a, AttributeValueSlice(values))
if len(values) == 1 {
o.Set(a, AttributeValueOne{values[0]})
} else {
o.Set(a, AttributeValueSlice(values))
}
}

func (o *Object) SetFlex(flexinit ...interface{}) {
Expand All @@ -698,6 +703,7 @@ func (o *Object) setFlex(flexinit ...interface{}) {
attribute := NonExistingAttribute

data := avsPool.Get().(AttributeValueSlice)

for _, i := range flexinit {
if i == IgnoreBlanks {
ignoreblanks = true
Expand Down Expand Up @@ -782,6 +788,13 @@ func (o *Object) setFlex(flexinit ...interface{}) {
data = append(data, v)
case AttributeValueOne:
data = append(data, v.Value)
case []AttributeValue:
for _, value := range v {
if ignoreblanks && value.IsZero() {
continue
}
data = append(data, value)
}
case AttributeValueSlice:
for _, value := range v {
if ignoreblanks && value.IsZero() {
Expand All @@ -793,26 +806,30 @@ func (o *Object) setFlex(flexinit ...interface{}) {
// Ignore it
case Attribute:
if attribute != NonExistingAttribute && (!ignoreblanks || len(data) > 0) {
newdata := make(AttributeValueSlice, len(data))
copy(newdata, data)
o.set(attribute, newdata)

data = data[:0]
if len(data) == 1 {
o.set(attribute, AttributeValueOne{data[0]})
} else {
newdata := make(AttributeValueSlice, len(data))
copy(newdata, data)
o.set(attribute, newdata)
}
}
data = data[:0]
attribute = v
default:
panic("SetFlex called with invalid type in object declaration")
}
}
if attribute != NonExistingAttribute && (!ignoreblanks || len(data) > 0) {
newdata := make(AttributeValueSlice, len(data))
copy(newdata, data)
o.set(attribute, newdata)
}
if len(data) > 0 {
data = data[:0]
if len(data) == 1 {
o.set(attribute, AttributeValueOne{data[0]})
} else {
newdata := make(AttributeValueSlice, len(data))
copy(newdata, data)
o.set(attribute, newdata)
}
}
avsPool.Put(data)
avsPool.Put(data[:0])
}

func (o *Object) Set(a Attribute, values AttributeValues) {
Expand Down Expand Up @@ -871,30 +888,38 @@ func (o *Object) set(a Attribute, values AttributeValues) {
}

// Deduplication of values
valueslice := values.Slice()
for i, value := range valueslice {
switch avs := value.(type) {
switch vs := values.(type) {
case AttributeValueSlice:
if len(vs) == 1 {
log.Error().Msg("Wrong type")
}
for i, value := range vs {
switch avs := value.(type) {
case AttributeValueSID:
vs[i] = AttributeValueSID(dedup.D.S(string(avs)))
case AttributeValueString:
vs[i] = AttributeValueString(dedup.D.S(string(avs)))
case AttributeValueBlob:
vs[i] = AttributeValueBlob(dedup.D.S(string(avs)))
}
}
case AttributeValueOne:
switch avs := vs.Value.(type) {
case AttributeValueSID:
vs.Value = AttributeValueSID(dedup.D.S(string(avs)))
case AttributeValueString:
valueslice[i] = AttributeValueString(stringdedup.S(string(avs)))
vs.Value = AttributeValueString(dedup.D.S(string(avs)))
case AttributeValueBlob:
valueslice[i] = AttributeValueBlob(stringdedup.B([]byte(avs)))
vs.Value = AttributeValueBlob(dedup.D.S(string(avs)))
}
}

var av AttributeValues

if len(valueslice) == 1 {
av = AttributeValueOne{valueslice[0]}
} else {
av = AttributeValueSlice(valueslice)
}

if attributenums[a].onset != nil {
attributenums[a].onset(o, a, av)
o.values.Set(a, nil) // placeholder for iteration over attributes that are set
} else {
o.values.Set(a, av)
}
// if attributenums[a].onset != nil {
// attributenums[a].onset(o, a, av)
// o.values.Set(a, nil) // placeholder for iteration over attributes that are set
// } else {
o.values.Set(a, values)
// }
}

func (o *Object) Meta() map[string]string {
Expand Down
Loading

0 comments on commit c1aeb3d

Please sign in to comment.