Got rid of my own stringdedup package, moved names before ldap query …

…in nodefilter, moved order by, limit, skip after ldap query in nodefilter, fixed limit/skip, optimized case sensitive searches in node filtering, renamed "Sample AQL Queries" to "AQL Queries", fixed lexer problem

go.mod

@@ -19,7 +19,6 @@ require (
 	github.com/lkarlslund/binstruct v1.3.1-0.20220418073417-7618823b3136
 	github.com/lkarlslund/go-win64api v0.0.0-20211005130710-d4f2d07ed091
 	github.com/lkarlslund/ldap/v3 v3.4.4-2.0.20240213092859-7df1bd8df6a8
-	github.com/lkarlslund/stringdedup v0.6.3-0.20231023091703-cf3e4d8e8ab3
 	github.com/lkarlslund/stringsplus v0.0.0-20211104080454-45e60fe6edc0
 	github.com/lkarlslund/time-timespan v0.0.0-20210712111050-6e7c565fa001
 	github.com/mailru/easyjson v0.7.7
@@ -75,7 +74,6 @@ require (
 	github.com/ugorji/go/codec v1.2.12
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
-	go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
 	golang.org/x/crypto v0.29.0 // indirect
 	golang.org/x/net v0.31.0 // indirect
 	google.golang.org/protobuf v1.35.2 // indirect
@@ -96,6 +94,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/jcmturner/gokrb5/v8 v8.4.4
 	github.com/lkarlslund/gonk v0.0.0-20241109090940-dffca74316e8
+	github.com/puzpuzpuz/xsync v1.5.2
 	github.com/spf13/viper v1.19.0
 	github.com/timtadh/lexmachine v0.2.3
 	go.etcd.io/bbolt v1.3.11
@@ -106,7 +105,6 @@ require github.com/timtadh/data-structures v0.6.1 // indirect
 
 require (
 	atomicgo.dev/schedule v0.1.0 // indirect
-	github.com/Cyan4973/xxHash v0.8.2 // indirect
 	github.com/DataDog/gostackparse v0.7.0 // indirect
 	github.com/StackExchange/wmi v1.2.0 // indirect
 	github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect
@@ -133,7 +131,6 @@ require (
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
-	github.com/puzpuzpuz/xsync v1.5.2 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
@@ -145,9 +142,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.12.0 // indirect
 	golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect
-	golang.org/x/mod v0.20.0 // indirect
-	golang.org/x/sync v0.9.0 // indirect
-	golang.org/x/tools v0.24.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

modules/aql/api.go

@@ -76,7 +76,7 @@ func init() {
 				}
 			}
 			if prunedislands > 0 {
-				ui.Debug().Msgf("Pruning islands removed %v nodes, leaving %v nodes", prunedislands, results.Order())
+				ui.Info().Msgf("Pruning islands removed %v nodes, leaving %v nodes", prunedislands, results.Order())
 			}
 
 			var objecttypes [256]int

modules/aql/lexer.go

@@ -19,17 +19,8 @@ var Lexer *lexmachine.Lexer // The lexer object. Use this to construct a Scanner
 
 const (
 	Invalid TokenID = iota
-	Integer
-	Float
-
-	UnquotedLDAPString
-	QuotedString // Quoted string
-
-	Identifier
-	HashIdentifier
-	AtIdentifier
 
-	Comment
+	// ORDERING MATTERS!
 
 	Star
 	Slash
@@ -72,6 +63,7 @@ const (
 	Match
 	Where
 	Skip
+	Offset
 	Limit
 	OrderBy
 	Desc
@@ -84,6 +76,20 @@ const (
 	Keyword
 
 	Whitespace
+
+	Integer
+	Float
+
+	UnquotedLDAPString
+	QuotedString // Quoted string
+
+	Identifier
+	HashIdentifier
+	AtIdentifier
+
+	Comment
+
+	MAXTOKEN = Comment
 )
 
 var StaticLexers = map[string]TokenID{
@@ -127,14 +133,15 @@ var StaticLexers = map[string]TokenID{
 	"\\-\\>": EdgeOut,
 	"\\<\\-": EdgeIn,
 
-	"MATCH ":    Match,
-	" IS":       Is,
-	" WHERE":    Where,
-	" SKIP":     Skip,
-	" LIMIT":    Limit,
-	" ORDER BY": OrderBy,
-	" DESC":     Desc,
-	" UNION":    Union,
+	"MATCH":    Match,
+	"IS":       Is,
+	"WHERE":    Where,
+	"SKIP":     Skip,
+	"OFFSET":   Offset,
+	"LIMIT":    Limit,
+	"ORDER BY": OrderBy,
+	"DESC":     Desc,
+	"UNION":    Union,
 
 	`//[^\n]*\n?`: Comment,
 	`/\*([^*]|\r|\n|(\*+([^*/]|\r|\n)))*\*+/`: Comment,
@@ -149,8 +156,13 @@ var StaticLexers = map[string]TokenID{
 func getLexer() (*lexmachine.Lexer, error) {
 	lexer := lexmachine.NewLexer()
 
-	for autolex, id := range StaticLexers {
-		lexer.Add([]byte(autolex), tokenid(id))
+	// Preserve ordering in the dumbest way possible
+	for currentid := range MAXTOKEN {
+		for autolex, id := range StaticLexers {
+			if currentid+1 == id {
+				lexer.Add([]byte(autolex), tokenid(id))
+			}
+		}
 	}
 
 	// lexer.Add([]byte(`([^)]|(\\.))+`),

modules/aql/parser.go

@@ -20,7 +20,7 @@ func ParseAQLQuery(s string, ao *engine.Objects) (AQLresolver, error) {
 	}
 	resolver, err := parseAQLstream(ts, ao)
 	if err != nil {
-		return nil, fmt.Errorf("parsing error: %v around %v", err, ts.Token().Position)
+		return nil, fmt.Errorf("parsing error: %v around position %v", err, ts.Token().Position.TC)
 	}
 	return resolver, nil
 }
@@ -52,7 +52,7 @@ func parseAQLquery(ts *TokenStream, ao *engine.Objects) (AQLresolver, error) {
 		Mode:       Acyclic, // default to something sane
 	}
 
-	for ts.Token().Is(Identifier) {
+	for ts.Token().Is(Identifier) && ts.PeekNextRawToken().Is(Whitespace) {
 		switch strings.ToUpper(ts.Token().Value) {
 		case "WALK":
 			result.Mode = Walk // Say goodbye to your CPU
@@ -102,16 +102,16 @@ func parseAQLquery(ts *TokenStream, ao *engine.Objects) (AQLresolver, error) {
 func parseNodeFilter(ts *TokenStream, ao *engine.Objects) (NodeQuery, error) {
 	var result NodeQuery
 
-	if !ts.NextIfIs(LParan) {
-		return NodeQuery{}, errors.New("Expecting ( as start of node query")
-	}
-
 	if ts.Token().Type == Identifier && (ts.PeekNextToken().Type == Colon || ts.PeekNextToken().Type == Is) {
 		result.ReferenceName = ts.Token().Value
 		ts.Next()
 		ts.Next()
 	}
 
+	if !ts.NextIfIs(LParan) {
+		return NodeQuery{}, errors.New("Expecting ( as start of node query")
+	}
+
 	// If RParan there is no selector, just select everything
 	if !ts.NextIfIs(RParan) {
 		where, err := parseLDAPFilterUnwrapped(ts, ao)
@@ -120,39 +120,45 @@ func parseNodeFilter(ts *TokenStream, ao *engine.Objects) (NodeQuery, error) {
 		}
 		result.Selector = where
 
-		// If we parse ORDER BY, it's fine
-		sorter, err := parseNodeSorter(ts, ao)
-		if err != nil {
-			return result, err
+		if !ts.NextIfIs(RParan) {
+			return NodeQuery{}, errors.New("Expecting ) at end of LDAP filter")
 		}
-		result.OrderBy = sorter // might be nil, if there was none
 
-		// If we parse a SKIP, it's fine
-		if ts.NextIfIs(Skip) {
-			skip := ts.Token()
-			if skip.Type != Integer {
-				return result, fmt.Errorf("SKIP value expects Integer, but I got %v (%v)", skip.Type, skip.Value)
-			}
+	}
 
-			result.Skip = int(skip.Native.(int64))
-			ts.Next()
-		}
+	// If we parse ORDER BY, it's fine
+	sorter, err := parseNodeSorter(ts, ao)
+	if err != nil {
+		return result, err
+	}
+	result.OrderBy = sorter // might be nil, if there was none
 
-		if ts.NextIfIs(Limit) {
-			limit := ts.Token()
-			if limit.Type != Integer {
-				return result, fmt.Errorf("LIMIT value expects Integer, but I got %v", limit.Type)
-			}
-			result.Limit = int(limit.Native.(int64))
+	// If we parse a SKIP, it's fine
+	if ts.NextIfIs(Skip) || ts.NextIfIs(Offset) {
+		skip := ts.Token()
+		if skip.Type != Integer {
+			return result, fmt.Errorf("SKIP value expects Integer, but I got %v (%v)", skip.Type, skip.Value)
+		}
 
-			ts.Next()
+		result.Skip = int(skip.Native.(int64))
+		if result.Skip == 0 {
+			return result, fmt.Errorf("SKIP value expects Integer > 0 or Integer < 0, but I got %v", skip.Value)
 		}
 
-		if !ts.NextIfIs(RParan) {
-			return NodeQuery{}, errors.New("Expecting ) at end of LDAP filter")
+		ts.Next()
+	}
+
+	if ts.NextIfIs(Limit) {
+		limit := ts.Token()
+		if limit.Type != Integer {
+			return result, fmt.Errorf("LIMIT value expects Integer, but I got %v", limit.Type)
 		}
-	} else {
-		result.Selector = nil // empty selector
+		result.Limit = int(limit.Native.(int64))
+		if result.Limit == 0 {
+			return result, fmt.Errorf("LIMIT value expects Integer > 0 or Integer < 0, but I got %v", limit.Value)
+		}
+
+		ts.Next()
 	}
 
 	return result, nil
@@ -526,7 +532,7 @@ func parseLDAPFilterUnwrapped(ts *TokenStream, ao *engine.Objects) (query.NodeFi
 			} else {
 				result = genwrapper(query.HasStringMatch{
 					Casesensitive: casesensitive,
-					Value:         strval})
+					Value:         engine.NewAttributeValueString(strval)})
 			}
 		}
 	}