Create storage conversion helper functions (#1495)

* Move supporting types into subpackage

* Move type visitor creation onto StorageTypeFactory

* Remove StorageTypesVisitorContext

* Rename all receivers

* Use AsEnumType() instead of hard cast

* Move creation of storage types into the factory

* Use a separate factory for each group

* Fix bug generating storage versions of resources

* Use queue for just in time creation of storage types

* Improve logging of conversion issues

* Don't case convert endpoint names

* Update StorageTypeFactory to use the name queue

* Code gardening

* Add injection of conversion functions

* Improve debug diagnostics

* Change to using TypeVisitorBuilder

* Code gardening

* Rename member to otherDefinition

* Use PropertyContainer to generate conversions

* Code gardening

* Move visitor methods to end of file

* Create and use visitor to inject functions

* Code gardening

* Make TypeFlag.IsOn() handled nested types

* Implement AsZero for ValidatedType

* Change implementation of type conversion skipping

* Code gardening

* Clear PackageReferenceSet

* Address issues identified by CI build

* Modify generator to reuse the err local in generated code

* Remove variable shadowing

* Resources need to use function references too

* Ensure "errors" is imported

* Use nested conversion contexts where needed

* Use pointer receiver for converion types

* Pass address of local

* Exclude storage version types from hub version selection

* Update from k8s-infra PR

* Create FunctionInjector abstraction

* Create HubVersionMarker abstraction

* Create PropertyConverter abstraction

* Create TypeConverter abstraction

* Add required headers

* Fix references

* go fmt

* Fix bug with missing KnownLocalsSet in context

* Ensure pointers are captured, not values

* Add Process() to Types

* Preserve ResourceReferences

* Code gardening

* Add support for ResourceReference properties

* Simplify error handling when creating conversions

* Refactor StorageTypeFactory

* Add warning

* Simplify handling of aggregate errors

* Rename member to group

* Include property type in error

* Go fmt

* Apply suggestions from code review

Co-authored-by: Matthew Christopher <[email protected]>

* Fix terminology, using package instead of namespace

* Add GitHub errors reference

Co-authored-by: Matthew Christopher <[email protected]>

hack/generator/pkg/astbuilder/builder.go

@@ -303,9 +303,10 @@ func ReturnNoError() dst.Stmt {
 //
 // errors.Wrap(err, <message>)
 //
-func WrappedErrorf(template string, args ...interface{}) dst.Expr {
+// (actual package name will be used, which will usually be 'errors')
+func WrappedErrorf(errorsPackage string, template string, args ...interface{}) dst.Expr {
 	return CallQualifiedFunc(
-		"errors",
+		errorsPackage,
 		"Wrap",
 		dst.NewIdent("err"),
 		StringLiteralf(template, args...))

hack/generator/pkg/astmodel/known_locals_set.go

@@ -79,6 +79,7 @@ func (locals *KnownLocalsSet) Add(local string) {
 	locals.names[name] = struct{}{}
 }
 
+// HasName returns true if the specified name exists in the set, false otherwise
 func (locals *KnownLocalsSet) HasName(name string) bool {
 	_, ok := locals.names[name]
 	return ok

hack/generator/pkg/astmodel/optional_type.go

@@ -104,7 +104,7 @@ func (optional *OptionalType) BaseType() Type {
 
 // String implements fmt.Stringer
 func (optional *OptionalType) String() string {
-	return fmt.Sprintf("(optional: %s)", optional.element.String())
+	return fmt.Sprintf("optional(%s)", optional.element.String())
 }
 
 // Unwrap returns the type contained within the wrapper type

hack/generator/pkg/astmodel/package_reference.go

@@ -48,15 +48,19 @@ func PackageAsLocalPackage(pkg PackageReference) (LocalPackageReference, error)
 
 func SortPackageReferencesByPathAndVersion(packages []PackageReference) {
 	sort.Slice(packages, func(i, j int) bool {
-		comparer := versionComparer{
-			left:  []rune(packages[i].PackagePath()),
-			right: []rune(packages[j].PackagePath()),
-		}
-		compare := comparer.Compare()
-		return compare < 0
+		return ComparePathAndVersion(packages[i].PackagePath(), packages[j].PackagePath())
 	})
 }
 
+// ComparePathAndVersion compares two paths containing versions and returns true if left should go before right
+func ComparePathAndVersion(left string, right string) bool {
+	comparer := versionComparer{
+		left:  []rune(left),
+		right: []rune(right),
+	}
+	return comparer.Compare() < 0
+}
+
 // versionComparer captures our state while doing an alphanumeric version comparision
 // We need separate indexes for each side because we're doing a numeric comparison, which will
 // compare "100" and "0100" as equal (leading zeros are not significant)

hack/generator/pkg/astmodel/package_reference_set.go

@@ -43,6 +43,11 @@ func (set *PackageReferenceSet) Remove(ref PackageReference) {
 	delete(set.references, ref)
 }
 
+// Clear removes everything from the set
+func (set *PackageReferenceSet) Clear() {
+	set.references = make(map[PackageReference]struct{})
+}
+
 // Contains allows checking to see if an import is included
 func (set *PackageReferenceSet) Contains(ref PackageReference) bool {
 	_, ok := set.references[ref]

hack/generator/pkg/astmodel/resource_type.go

@@ -362,6 +362,10 @@ func (resource *ResourceType) RequiredPackageReferences() *PackageReferenceSet {
 		references.Merge(resource.status.RequiredPackageReferences())
 	}
 
+	for _, fn := range resource.functions {
+		references.Merge(fn.RequiredPackageReferences())
+	}
+
 	// Interface imports
 	references.Merge(resource.InterfaceImplementer.RequiredPackageReferences())
 

hack/generator/pkg/astmodel/std_references.go

@@ -7,14 +7,14 @@ package astmodel
 
 var (
 	// References to standard Go Libraries
-	ErrorsReference  PackageReference = MakeExternalPackageReference("errors")
-	FmtReference     PackageReference = MakeExternalPackageReference("fmt")
-	JsonReference    PackageReference = MakeExternalPackageReference("encoding/json")
-	ReflectReference PackageReference = MakeExternalPackageReference("reflect")
-	TestingReference PackageReference = MakeExternalPackageReference("testing")
+	ErrorsReference  = MakeExternalPackageReference("errors")
+	FmtReference     = MakeExternalPackageReference("fmt")
+	JsonReference    = MakeExternalPackageReference("encoding/json")
+	ReflectReference = MakeExternalPackageReference("reflect")
+	TestingReference = MakeExternalPackageReference("testing")
 
 	// References to our Libraries
-	GenRuntimeReference PackageReference = MakeExternalPackageReference(genRuntimePathPrefix)
+	GenRuntimeReference = MakeExternalPackageReference(genRuntimePathPrefix)
 
 	// References to other libraries
 	APIExtensionsReference       = MakeExternalPackageReference("k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1")
@@ -23,19 +23,20 @@ var (
 	APIMachineryRuntimeReference = MakeExternalPackageReference("k8s.io/apimachinery/pkg/runtime")
 	ClientGoSchemeReference      = MakeExternalPackageReference("k8s.io/client-go/kubernetes/scheme")
 	ControllerRuntimeAdmission   = MakeExternalPackageReference("sigs.k8s.io/controller-runtime/pkg/webhook/admission")
+	GitHubErrorsReference        = MakeExternalPackageReference("github.com/pkg/errors")
 
 	// References to libraries used for testing
-	CmpReference        PackageReference = MakeExternalPackageReference("github.com/google/go-cmp/cmp")
-	CmpOptsReference    PackageReference = MakeExternalPackageReference("github.com/google/go-cmp/cmp/cmpopts")
-	DiffReference       PackageReference = MakeExternalPackageReference("github.com/kylelemons/godebug/diff")
-	GopterReference     PackageReference = MakeExternalPackageReference("github.com/leanovate/gopter")
-	GopterGenReference  PackageReference = MakeExternalPackageReference("github.com/leanovate/gopter/gen")
-	GopterPropReference PackageReference = MakeExternalPackageReference("github.com/leanovate/gopter/prop")
-	GomegaReference     PackageReference = MakeExternalPackageReference("github.com/onsi/gomega")
-	PrettyReference     PackageReference = MakeExternalPackageReference("github.com/kr/pretty")
+	CmpReference        = MakeExternalPackageReference("github.com/google/go-cmp/cmp")
+	CmpOptsReference    = MakeExternalPackageReference("github.com/google/go-cmp/cmp/cmpopts")
+	DiffReference       = MakeExternalPackageReference("github.com/kylelemons/godebug/diff")
+	GopterReference     = MakeExternalPackageReference("github.com/leanovate/gopter")
+	GopterGenReference  = MakeExternalPackageReference("github.com/leanovate/gopter/gen")
+	GopterPropReference = MakeExternalPackageReference("github.com/leanovate/gopter/prop")
+	GomegaReference     = MakeExternalPackageReference("github.com/onsi/gomega")
+	PrettyReference     = MakeExternalPackageReference("github.com/kr/pretty")
 
 	// Imports with specified names
-	GomegaImport PackageImport = NewPackageImport(GomegaReference).WithName(".")
+	GomegaImport = NewPackageImport(GomegaReference).WithName(".")
 
 	// Type names
 	ResourceReferenceTypeName = MakeTypeName(GenRuntimeReference, "ResourceReference")

hack/generator/pkg/astmodel/storage_conversion_context.go

@@ -12,21 +12,40 @@ type StorageConversionContext struct {
 	types Types
 	// functionName is the name of the function we're currently generating
 	functionName string
+	// knownLocals is a reference to our set of local variables
+	// (Pointer because it's a reference type, not because it's optional)
+	knownLocals *KnownLocalsSet
 }
 
 // NewStorageConversionContext creates a new instance of a StorageConversionContext
-func NewStorageConversionContext(types Types) *StorageConversionContext {
+func NewStorageConversionContext(types Types, idFactory IdentifierFactory) *StorageConversionContext {
 	return &StorageConversionContext{
-		types: types,
+		types:       types,
+		knownLocals: NewKnownLocalsSet(idFactory),
 	}
 }
 
+// WithFunctionName returns a new context with the specified function name included
 func (c *StorageConversionContext) WithFunctionName(name string) *StorageConversionContext {
-	result := NewStorageConversionContext(c.types)
+	result := c.clone()
 	result.functionName = name
 	return result
 }
 
+// WithKnownLocals returns a new context with the specified known locals set referenced
+func (c *StorageConversionContext) WithKnownLocals(knownLocals *KnownLocalsSet) *StorageConversionContext {
+	result := c.clone()
+	result.knownLocals = knownLocals
+	return result
+}
+
+// NestedContext returns a new context with a cloned knownLocals so that nested blocks
+// can declare and reuse locals independently of each other.
+func (c *StorageConversionContext) NestedContext() *StorageConversionContext {
+	result := c.clone()
+	return result
+}
+
 // ResolveType resolves a type that might be a type name into both the name and the actual
 // type it references, returning true iff it was a TypeName that could be resolved
 func (c *StorageConversionContext) ResolveType(t Type) (TypeName, Type, bool) {
@@ -42,3 +61,22 @@ func (c *StorageConversionContext) ResolveType(t Type) (TypeName, Type, bool) {
 
 	return name, actualType, true
 }
+
+// TryCreateLocal returns true if the specified local was successfully created, false if it already existed
+func (c *StorageConversionContext) TryCreateLocal(local string) bool {
+	if c.knownLocals.HasName(local) {
+		return false
+	}
+
+	c.knownLocals.Add(local)
+	return true
+}
+
+// clone returns a new independent copy of this context
+func (c *StorageConversionContext) clone() *StorageConversionContext {
+	return &StorageConversionContext{
+		types:        c.types,
+		functionName: c.functionName,
+		knownLocals:  c.knownLocals.Clone(),
+	}
+}

hack/generator/pkg/astmodel/storage_conversion_endpoint.go

@@ -6,8 +6,6 @@
 package astmodel
 
 import (
-	"strings"
-
 	"github.com/gobuffalo/flect"
 )
 
@@ -28,7 +26,7 @@ func NewStorageConversionEndpoint(
 	knownLocals *KnownLocalsSet) *StorageConversionEndpoint {
 	return &StorageConversionEndpoint{
 		theType:     theType,
-		name:        strings.ToLower(name),
+		name:        name,
 		knownLocals: knownLocals,
 	}
 }