Add walk package to traverse an AST (#265)

* Add `walk` package to traverse an AST

- Add `ast-gen-walk`
- Remove `ColumnDefOptions` and `VectorIndexOption` because they are unused.

* Move `Walk` functions to `ast` package

* Fix format

* Fix BinaryExpr.end

* Do make gen

* Add pos-end test into parser_test.go

* Fix Join.end

* Fix Selector.end

* Fix AlterDatabase.end

* Fix CreateTable.end

* Fix AlterIndex.end

* Fix end of ChangeStream

* Fix TableConstraint.ConstraintPos

* Make gen

* go test --update

* Fix interval notation

* Update pos/end test

* Add `Field` and `Index` to `Visitor`

* Add `WalkMany` and other `Many` variants

* Check and fix the field order

* Update conventions

* Update ast/ast.go

Co-authored-by: apstndb <[email protected]>

* Update Go version on CI

* Update walk_internal.go

* Fix PropertyGraphEdgeElementKeys pos

* Insert an empty line

---------

Co-authored-by: apstndb <[email protected]>

.github/workflows/go.yml

@@ -11,10 +11,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.20
+    - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: "1.22"
+        go-version: "1.23"
       id: go
 
     - name: Check out code into the Go module directory

ast/ast.go

@@ -39,11 +39,12 @@ package ast
 //
 // Conventions:
 //
-//   - Each node interface (except for Node) should have isXXX method (XXX must be a name of the interface itself).
-//   - `isXXX` methods should be defined after the interface definition
-//     and the receiver should be the non-pointer node struct type.
-//   - Each node struct should have pos and end comments.
-//   - Each node struct should have template lines in its doc comment.
+//   - Each node interface (except for Node) must have isXXX method (XXX is a name of the interface itself).
+//   - `isXXX` methods must be defined after the interface definition
+//     and the receiver must be the non-pointer node struct type.
+//   - Each node struct must have pos and end comments.
+//   - Each node struct must have template lines in its doc comment.
+//   - The fields of each node must be ordered by the position.
 
 //go:generate go run ../tools/gen-ast-pos/main.go -astfile ast.go -constfile ast_const.go -outfile pos.go
 
@@ -1177,12 +1178,13 @@ type ParenTableExpr struct {
 //	{{.Cond | sqlOpt}}
 type Join struct {
 	// pos = Left.pos
-	// end = (Cond ?? Right).pos
+	// end = (Cond ?? Right).end
 
-	Op          JoinOp
-	Method      JoinMethod
-	Hint        *Hint // optional
-	Left, Right TableExpr
+	Left   TableExpr
+	Op     JoinOp
+	Method JoinMethod
+	Hint   *Hint // optional
+	Right  TableExpr
 
 	// nil when Op is CrossJoin
 	// optional when Right is PathTableExpr or Unnest
@@ -1252,7 +1254,7 @@ type TableSampleSize struct {
 //	{{.Left | sql}} {{.Op}} {{.Right | sql}}
 type BinaryExpr struct {
 	// pos = Left.pos
-	// end = Right.pos
+	// end = Right.end
 
 	Op BinaryOp
 
@@ -1364,7 +1366,7 @@ type BetweenExpr struct {
 //	{{.Expr | sql}}.{{.Ident | sql}}
 type SelectorExpr struct {
 	// pos = Expr.pos
-	// end = Ident.pos
+	// end = Ident.end
 
 	Expr  Expr
 	Ident *Ident
@@ -2255,7 +2257,7 @@ type CreateDatabase struct {
 //	ALTER DATABASE {{.Name | sql}} SET {{.Options | sql}}
 type AlterDatabase struct {
 	// pos = Alter
-	// end = Name.end
+	// end = Options.end
 
 	Alter token.Pos // position of "ALTER" keyword
 
@@ -2383,10 +2385,11 @@ type DropProtoBundle struct {
 // the original order of them, please sort them by their `Pos()`.
 type CreateTable struct {
 	// pos = Create
-	// end = RowDeletionPolicy.end || Cluster.end || Rparen + 1
+	// end = RowDeletionPolicy.end || Cluster.end || PrimaryKeyRparen + 1 || Rparen + 1
 
-	Create token.Pos // position of "CREATE" keyword
-	Rparen token.Pos // position of ")" of PRIMARY KEY clause
+	Create           token.Pos // position of "CREATE" keyword
+	Rparen           token.Pos // position of ")" of end of column definitions
+	PrimaryKeyRparen token.Pos // position of ")" of PRIMARY KEY clause, optional
 
 	IfNotExists       bool
 	Name              *Path
@@ -2530,19 +2533,6 @@ type IdentityColumn struct {
 	Params []SequenceParam //  if Rparen.Invalid() then len(Param) = 0 else len(Param) > 0
 }
 
-// ColumnDefOption is options for column definition.
-//
-//	OPTIONS(allow_commit_timestamp = {{if .AllowCommitTimestamp}}true{{else}null{{end}}})
-type ColumnDefOptions struct {
-	// pos = Options
-	// end = Rparen + 1
-
-	Options token.Pos // position of "OPTIONS" keyword
-	Rparen  token.Pos // position of ")"
-
-	AllowCommitTimestamp bool
-}
-
 // TableConstraint is table constraint in CREATE TABLE and ALTER TABLE.
 //
 //	{{if .Name}}CONSTRAINT {{.Name}}{{end}}{{.Constraint | sql}}
@@ -2703,7 +2693,7 @@ type AlterIndex struct {
 //	{{.NoSkipRange | sqlOpt}}
 type AlterSequence struct {
 	// pos = Alter
-	// end = Options.end
+	// end = (NoSkipRange ?? SkipRange ?? RestartCounterWith ?? Options).end
 
 	Alter token.Pos // position of "ALTER" keyword
 
@@ -3062,17 +3052,6 @@ type CreateVectorIndex struct {
 	Options *Options
 }
 
-// VectorIndexOption is OPTIONS record node.
-//
-//	{{.Key | sql}}={{.Expr | sql}}
-type VectorIndexOption struct {
-	// pos = Key.pos
-	// end = Value.end
-
-	Key   *Ident
-	Value Expr
-}
-
 // CreateChangeStream is CREATE CHANGE STREAM statement node.
 //
 //	CREATE CHANGE STREAM {{.Name | sql}} {{.For | sqlOpt}} {{.Options | sqlOpt}}
@@ -3092,7 +3071,7 @@ type CreateChangeStream struct {
 //	FOR ALL
 type ChangeStreamForAll struct {
 	// pos = For
-	// end = All
+	// end = All + 3
 
 	For token.Pos // position of "FOR" keyword
 	All token.Pos // position of "ALL" keyword
@@ -3115,7 +3094,7 @@ type ChangeStreamForTables struct {
 //	{{.TableName | sql}}{{if .Columns}}({{.Columns | sqlJoin ","}}){{end}}
 type ChangeStreamForTable struct {
 	// pos = TableName.pos
-	// end = TableName.end || Rparen + 1
+	// end = Rparen + 1 || TableName.end
 
 	Rparen token.Pos // position of ")"
 
@@ -3808,8 +3787,9 @@ type PropertyGraphNodeElementKey struct {
 //
 //	{{.Element | sqlOpt}} {{.Source | sql}} {{.Destination | sql}}
 type PropertyGraphEdgeElementKeys struct {
-	// pos = Element.pos
+	// pos = (Element ?? Source).pos
 	// end = Destination.end
+
 	Element     *PropertyGraphElementKey // optional
 	Source      *PropertyGraphSourceKey
 	Destination *PropertyGraphDestinationKey

ast/node_wrapper.go

@@ -0,0 +1,22 @@
+package ast
+
+type comparableNode = interface {
+	comparable
+	Node
+}
+
+func wrapNode[T comparableNode](node T) Node {
+	var zero T
+	if node == zero {
+		return nil
+	}
+	return node
+}
+
+func wrapNodes[T Node](nodes []T) []Node {
+	result := make([]Node, 0, len(nodes))
+	for _, node := range nodes {
+		result = append(result, node)
+	}
+	return result
+}

ast/pos_util.go

@@ -6,11 +6,6 @@ import (
 
 // This file contains utility types/functions for pos.go.
 
-type comparableNode = interface {
-	comparable
-	Node
-}
-
 func nodePos(n Node) token.Pos {
 	if n == nil {
 		return token.InvalidPos
@@ -42,14 +37,6 @@ func posAdd(p token.Pos, x int) token.Pos {
 	return token.Pos(int(p) + x)
 }
 
-func wrapNode[T comparableNode](node T) Node {
-	var zero T
-	if node == zero {
-		return nil
-	}
-	return node
-}
-
 func nodeChoice(ns ...Node) Node {
 	for _, n := range ns {
 		if n != nil {