From 76193e5d5dc61eb2ffeb93cbde0b6146b4d41b9b Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Fri, 30 Oct 2020 08:42:14 +0100 Subject: [PATCH 1/3] codegen: rearrange output into finite number of files. Also, emit some comments around the type definitions. The old file layout is still available, but renamed to GenerateSplayed. It will probably be removed in the future. The new format does not currently have stable output order. I'd like to preserve the original order given by the schema, but our current placeholder types for schema data don't have this. More work needed on this. --- schema/gen/go/adjunctCfg.go | 5 ++ schema/gen/go/genList.go | 5 +- schema/gen/go/genMap.go | 5 +- schema/gen/go/genStruct.go | 5 +- schema/gen/go/genUnion.go | 6 +- schema/gen/go/generate.go | 113 ++++++++++++++++++++++++++++++++ schema/gen/go/generators.go | 9 +-- schema/gen/go/genpartsCommon.go | 5 +- schema/gen/go/genpartsMinima.go | 4 ++ schema/gen/go/templateUtil.go | 1 + 10 files changed, 148 insertions(+), 10 deletions(-) diff --git a/schema/gen/go/adjunctCfg.go b/schema/gen/go/adjunctCfg.go index 38c5df2c..d3d603fa 100644 --- a/schema/gen/go/adjunctCfg.go +++ b/schema/gen/go/adjunctCfg.go @@ -65,6 +65,11 @@ func (cfg *AdjunctCfg) FieldSymbolUpper(f schema.StructField) string { return strings.Title(f.Name()) } +// Comments returns a bool for whether comments should be included in gen output or not. +func (cfg *AdjunctCfg) Comments() bool { + return true // FUTURE: okay, maybe this should be configurable :) +} + func (cfg *AdjunctCfg) MaybeUsesPtr(t schema.Type) bool { if x, ok := cfg.maybeUsesPtr[t.Name()]; ok { return x diff --git a/schema/gen/go/genList.go b/schema/gen/go/genList.go index 32cbcb28..39eeb920 100644 --- a/schema/gen/go/genList.go +++ b/schema/gen/go/genList.go @@ -21,10 +21,13 @@ func (listGenerator) IsRepr() bool { return false } // hint used in some general func (g listGenerator) EmitNativeType(w io.Writer) { // Lists are a pretty straightforward struct enclosing a slice. doTemplate(` + {{- if Comments -}} + // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .ReprKind }} kind. + {{- end}} + type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { x []_{{ .Type.ValueType | TypeSymbol }}{{if .Type.ValueIsNullable }}__Maybe{{end}} } - type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } diff --git a/schema/gen/go/genMap.go b/schema/gen/go/genMap.go index ba31f0aa..3a73bcba 100644 --- a/schema/gen/go/genMap.go +++ b/schema/gen/go/genMap.go @@ -25,11 +25,14 @@ func (g mapGenerator) EmitNativeType(w io.Writer) { // Note that the key in 'm' is *not* a pointer. // The value in 'm' is a pointer into 't' (except when it's a maybe; maybes are already pointers). doTemplate(` + {{- if Comments -}} + // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .ReprKind }} kind. + {{- end}} + type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { m map[_{{ .Type.KeyType | TypeSymbol }}]{{if .Type.ValueIsNullable }}Maybe{{else}}*_{{end}}{{ .Type.ValueType | TypeSymbol }} t []_{{ .Type | TypeSymbol }}__entry } - type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) // - address of 'k' is used when we return keys as nodes, such as in iterators. // Having these in the 't' slice above amortizes moving all of them to heap at once, diff --git a/schema/gen/go/genStruct.go b/schema/gen/go/genStruct.go index c9867680..43a34ec8 100644 --- a/schema/gen/go/genStruct.go +++ b/schema/gen/go/genStruct.go @@ -20,12 +20,15 @@ func (structGenerator) IsRepr() bool { return false } // hint used in some gener func (g structGenerator) EmitNativeType(w io.Writer) { doTemplate(` + {{- if Comments -}} + // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Type.Kind }} type-kind, and may be interrogated like {{ .ReprKind }} kind. + {{- end}} + type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { {{- range $field := .Type.Fields}} {{ $field | FieldSymbolLower }} _{{ $field.Type | TypeSymbol }}{{if $field.IsMaybe }}__Maybe{{end}} {{- end}} } - type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} `, w, g.AdjCfg, g) } diff --git a/schema/gen/go/genUnion.go b/schema/gen/go/genUnion.go index ad9e9ab5..9dbc3076 100644 --- a/schema/gen/go/genUnion.go +++ b/schema/gen/go/genUnion.go @@ -45,6 +45,10 @@ func (g unionGenerator) EmitNativeType(w io.Writer) { // (see further comments in the EmitNodeAssemblerType function); // and since we do it in that one case, it's just as well to do it uniformly. doTemplate(` + {{- if Comments -}} + // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .Type.Kind }} type-kind, and may be interrogated like {{ .ReprKind }} kind. + {{- end}} + type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} type _{{ .Type | TypeSymbol }} struct { {{- if (eq (.AdjCfg.UnionMemlayout .Type) "embedAll") }} tag uint @@ -55,8 +59,6 @@ func (g unionGenerator) EmitNativeType(w io.Writer) { x _{{ .Type | TypeSymbol }}__iface {{- end}} } - type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} - type _{{ .Type | TypeSymbol }}__iface interface { _{{ .Type | TypeSymbol }}__member() } diff --git a/schema/gen/go/generate.go b/schema/gen/go/generate.go index 75e1454b..9de8bba8 100644 --- a/schema/gen/go/generate.go +++ b/schema/gen/go/generate.go @@ -1,6 +1,7 @@ package gengo import ( + "fmt" "io" "os" "path/filepath" @@ -8,7 +9,117 @@ import ( "github.com/ipld/go-ipld-prime/schema" ) +// Generate takes a typesystem and the adjunct config for codegen, +// and emits generated code in the given path with the given package name. +// +// All of the files produced will match the pattern "ipldsch.*.gen.go". func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg) { + // Emit fixed bits. + withFile(filepath.Join(pth, "ipldsch.minima.gen.go"), func(f io.Writer) { + EmitInternalEnums(pkgName, f) + }) + + // Local helper function for applying generation logic to each type. + // We will end up doing this more than once because in this layout, more than one file contains part of the story for each type. + applyToEachType := func(fn func(tg TypeGenerator, w io.Writer), f io.Writer) { + // FIXME: the order of this iteration is not stable, and it should be, because it affects determinism of the output. + for _, typ := range ts.GetTypes() { + switch t2 := typ.(type) { + case *schema.TypeBool: + fn(NewBoolReprBoolGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeInt: + fn(NewIntReprIntGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeFloat: + fn(NewFloatReprFloatGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeString: + fn(NewStringReprStringGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeBytes: + fn(NewBytesReprBytesGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeLink: + fn(NewLinkReprLinkGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeStruct: + switch t2.RepresentationStrategy().(type) { + case schema.StructRepresentation_Map: + fn(NewStructReprMapGenerator(pkgName, t2, adjCfg), f) + case schema.StructRepresentation_Tuple: + fn(NewStructReprTupleGenerator(pkgName, t2, adjCfg), f) + case schema.StructRepresentation_Stringjoin: + fn(NewStructReprStringjoinGenerator(pkgName, t2, adjCfg), f) + default: + panic("unrecognized struct representation strategy") + } + case *schema.TypeMap: + fn(NewMapReprMapGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeList: + fn(NewListReprListGenerator(pkgName, t2, adjCfg), f) + case *schema.TypeUnion: + switch t2.RepresentationStrategy().(type) { + case schema.UnionRepresentation_Keyed: + fn(NewUnionReprKeyedGenerator(pkgName, t2, adjCfg), f) + case schema.UnionRepresentation_Kinded: + fn(NewUnionReprKindedGenerator(pkgName, t2, adjCfg), f) + default: + panic("unrecognized union representation strategy") + } + default: + panic("add more type switches here :)") + } + } + } + + // Emit a file with the type table, and the golang type defns for each type. + withFile(filepath.Join(pth, "ipldsch.types.gen.go"), func(f io.Writer) { + // Emit headers, import statements, etc. + fmt.Fprintf(f, "package %s\n\n", pkgName) + fmt.Fprintf(f, doNotEditComment+"\n\n") + + // Emit the type table. + EmitTypeTable(pkgName, ts, adjCfg, f) + + // Emit the type defns matching the schema types. + fmt.Fprintf(f, "\n// --- type definitions follow ---\n\n") + applyToEachType(func(tg TypeGenerator, w io.Writer) { + tg.EmitNativeType(w) + fmt.Fprintf(f, "\n") + }, f) + + }) + + // Emit a file with all the Node/NodeBuilder/NodeAssembler boilerplate. + // Also includes typedefs for representation-level data. + // Also includes the MaybeT boilerplate. + withFile(filepath.Join(pth, "ipldsch.satisfaction.gen.go"), func(f io.Writer) { + // Emit headers, import statements, etc. + fmt.Fprintf(f, "package %s\n\n", pkgName) + fmt.Fprintf(f, doNotEditComment+"\n\n") + fmt.Fprintf(f, "import (\n") + fmt.Fprintf(f, "\tipld \"github.com/ipld/go-ipld-prime\"\n") // referenced everywhere. + fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/node/mixins\"\n") // referenced by node implementation guts. + fmt.Fprintf(f, "\t\"github.com/ipld/go-ipld-prime/schema\"\n") // referenced by maybes (and surprisingly little else). + fmt.Fprintf(f, ")\n\n") + + // For each type, we'll emit... everything except the native type, really. + applyToEachType(func(tg TypeGenerator, w io.Writer) { + tg.EmitNativeAccessors(w) + tg.EmitNativeBuilder(w) + tg.EmitNativeMaybe(w) + EmitNode(tg, w) + tg.EmitTypedNodeMethodType(w) + tg.EmitTypedNodeMethodRepresentation(w) + + nrg := tg.GetRepresentationNodeGen() + EmitNode(nrg, w) + + fmt.Fprintf(f, "\n") + }, f) + }) +} + +// GenerateSplayed is like Generate, but emits a differnet pattern of files. +// GenerateSplayed emits many more individual files than Generate. +// +// This function should be considered deprecated and may be removed in the future. +func GenerateSplayed(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg) { // Emit fixed bits. withFile(filepath.Join(pth, "minima.go"), func(f io.Writer) { EmitInternalEnums(pkgName, f) @@ -63,6 +174,8 @@ func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctC // Emit the unified type table. withFile(filepath.Join(pth, "typeTable.go"), func(f io.Writer) { + fmt.Fprintf(f, "package %s\n\n", pkgName) + fmt.Fprintf(f, doNotEditComment+"\n\n") EmitTypeTable(pkgName, ts, adjCfg, f) }) } diff --git a/schema/gen/go/generators.go b/schema/gen/go/generators.go index 2950ef34..5b23a610 100644 --- a/schema/gen/go/generators.go +++ b/schema/gen/go/generators.go @@ -90,8 +90,9 @@ func EmitFileHeader(packageName string, w io.Writer) { fmt.Fprintf(w, ")\n\n") } -// EmitEntireType calls all methods of TypeGenerator and streams -// all results into a single writer. +// EmitEntireType is a helper function calls all methods of TypeGenerator +// and streams all results into a single writer. +// (This implies two calls to EmitNode -- one for the type-level and one for the representation-level.) func EmitEntireType(tg TypeGenerator, w io.Writer) { tg.EmitNativeType(w) tg.EmitNativeAccessors(w) @@ -108,6 +109,8 @@ func EmitEntireType(tg TypeGenerator, w io.Writer) { EmitNode(rng, w) } +// EmitNode is a helper function that calls all methods of NodeGenerator +// and streams all results into a single writer. func EmitNode(ng NodeGenerator, w io.Writer) { ng.EmitNodeType(w) ng.EmitNodeTypeAssertions(w) @@ -156,8 +159,6 @@ func EmitTypeTable(pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg, w i // REVIEW: if "T__Repr" is how we want to expose this. We could also put 'Repr' accessors on the type/prototype objects. // FUTURE: types and prototypes are proposed to be the same. Some of this text pretends they already are, but work is needed on this. doTemplate(` - package `+pkgName+` - // Type is a struct embeding a NodePrototype/Type for every Node implementation in this package. // One of its major uses is to start the construction of a value. // You can use it like this: diff --git a/schema/gen/go/genpartsCommon.go b/schema/gen/go/genpartsCommon.go index 18b94335..055a4905 100644 --- a/schema/gen/go/genpartsCommon.go +++ b/schema/gen/go/genpartsCommon.go @@ -60,8 +60,11 @@ func emitNativeType_scalar(w io.Writer, adjCfg *AdjunctCfg, data interface{}) { // while also having the advantage of meaning we can block direct casting, // which is desirable because the compiler then ensures our validate methods can't be evaded. doTemplate(` - type _{{ .Type | TypeSymbol }} struct{ x {{ .ReprKind | KindPrim }} } + {{- if Comments -}} + // {{ .Type | TypeSymbol }} matches the IPLD Schema type "{{ .Type.Name }}". It has {{ .ReprKind }} kind. + {{- end}} type {{ .Type | TypeSymbol }} = *_{{ .Type | TypeSymbol }} + type _{{ .Type | TypeSymbol }} struct{ x {{ .ReprKind | KindPrim }} } `, w, adjCfg, data) } diff --git a/schema/gen/go/genpartsMinima.go b/schema/gen/go/genpartsMinima.go index 87d6f4cf..7153767e 100644 --- a/schema/gen/go/genpartsMinima.go +++ b/schema/gen/go/genpartsMinima.go @@ -10,6 +10,10 @@ import ( // EmitInternalEnums creates a file with enum types used internally. // For example, the state machine values used in map and list builders. // These always need to exist exactly once in each package created by codegen. +// +// The file header and import statements are included in the output of this function. +// (The imports in this file are different than most others in codegen output; +// we gather up any references to other packages in this file in order to simplify the rest of codegen's awareness of imports.) func EmitInternalEnums(packageName string, w io.Writer) { fmt.Fprint(w, wish.Dedent(` package `+packageName+` diff --git a/schema/gen/go/templateUtil.go b/schema/gen/go/templateUtil.go index ba6adab5..f9b7d874 100644 --- a/schema/gen/go/templateUtil.go +++ b/schema/gen/go/templateUtil.go @@ -20,6 +20,7 @@ func doTemplate(tmplstr string, w io.Writer, adjCfg *AdjunctCfg, data interface{ "FieldSymbolLower": adjCfg.FieldSymbolLower, "FieldSymbolUpper": adjCfg.FieldSymbolUpper, "MaybeUsesPtr": adjCfg.MaybeUsesPtr, + "Comments": adjCfg.Comments, // The whole AdjunctConfig can be accessed. // Access methods like UnionMemlayout through this, as e.g. `.AdjCfg.UnionMemlayout`. From 4f333954a5ffaf929c2f3d57b01cb7cc406470d8 Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Fri, 30 Oct 2020 18:57:30 +0100 Subject: [PATCH 2/3] codegen: deterministic order for types in output. I'd still probably prefer to replace this with simply having a stable order that is carried through consistently, but that remains blocked behind getting self-hosted types, and while it so happens I also got about 80% of the way there on those today, the second 80% may take another day. Better make this stable rather than wait. --- schema/gen/go/generate.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/schema/gen/go/generate.go b/schema/gen/go/generate.go index 9de8bba8..e5c6ef6f 100644 --- a/schema/gen/go/generate.go +++ b/schema/gen/go/generate.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "sort" "github.com/ipld/go-ipld-prime/schema" ) @@ -22,9 +23,16 @@ func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctC // Local helper function for applying generation logic to each type. // We will end up doing this more than once because in this layout, more than one file contains part of the story for each type. applyToEachType := func(fn func(tg TypeGenerator, w io.Writer), f io.Writer) { - // FIXME: the order of this iteration is not stable, and it should be, because it affects determinism of the output. - for _, typ := range ts.GetTypes() { - switch t2 := typ.(type) { + // Sort the type names so we have a determinisic order; this affects output consistency. + // Any stable order would do, but we don't presently have one, so a sort is necessary. + types := ts.GetTypes() + keys := make(sortableTypeNames, 0, len(types)) + for tn := range types { + keys = append(keys, tn) + } + sort.Sort(keys) + for _, tn := range keys { + switch t2 := types[tn].(type) { case *schema.TypeBool: fn(NewBoolReprBoolGenerator(pkgName, t2, adjCfg), f) case *schema.TypeInt: @@ -188,3 +196,9 @@ func withFile(filename string, fn func(io.Writer)) { defer f.Close() fn(f) } + +type sortableTypeNames []schema.TypeName + +func (a sortableTypeNames) Len() int { return len(a) } +func (a sortableTypeNames) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a sortableTypeNames) Less(i, j int) bool { return a[i] < a[j] } From 32e66f20c752120e2b3aea4ba1ff98a0868c7c1d Mon Sep 17 00:00:00 2001 From: Eric Myhre Date: Tue, 17 Nov 2020 21:53:45 +0100 Subject: [PATCH 3/3] codegen: rename files. An underscore; and less "gen", because reviewers indicated it felt redundant. --- schema/gen/go/generate.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/schema/gen/go/generate.go b/schema/gen/go/generate.go index e5c6ef6f..50915a33 100644 --- a/schema/gen/go/generate.go +++ b/schema/gen/go/generate.go @@ -16,7 +16,7 @@ import ( // All of the files produced will match the pattern "ipldsch.*.gen.go". func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctCfg) { // Emit fixed bits. - withFile(filepath.Join(pth, "ipldsch.minima.gen.go"), func(f io.Writer) { + withFile(filepath.Join(pth, "ipldsch_minima.go"), func(f io.Writer) { EmitInternalEnums(pkgName, f) }) @@ -76,7 +76,7 @@ func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctC } // Emit a file with the type table, and the golang type defns for each type. - withFile(filepath.Join(pth, "ipldsch.types.gen.go"), func(f io.Writer) { + withFile(filepath.Join(pth, "ipldsch_types.go"), func(f io.Writer) { // Emit headers, import statements, etc. fmt.Fprintf(f, "package %s\n\n", pkgName) fmt.Fprintf(f, doNotEditComment+"\n\n") @@ -96,7 +96,7 @@ func Generate(pth string, pkgName string, ts schema.TypeSystem, adjCfg *AdjunctC // Emit a file with all the Node/NodeBuilder/NodeAssembler boilerplate. // Also includes typedefs for representation-level data. // Also includes the MaybeT boilerplate. - withFile(filepath.Join(pth, "ipldsch.satisfaction.gen.go"), func(f io.Writer) { + withFile(filepath.Join(pth, "ipldsch_satisfaction.go"), func(f io.Writer) { // Emit headers, import statements, etc. fmt.Fprintf(f, "package %s\n\n", pkgName) fmt.Fprintf(f, doNotEditComment+"\n\n")