Skip to content

Commit

Permalink
Merge branch 'master' into improve/type-walker
Browse files Browse the repository at this point in the history
  • Loading branch information
@theunrepentantgeek
theunrepentantgeek authored Jun 25, 2021
2 parents af6ce1b + f40b9a7 commit f549305
Show file tree
Hide file tree
Showing 53 changed files with 516 additions and 445 deletions.
143 changes: 66 additions & 77 deletions hack/generator/pkg/codegen/code_generator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,24 +14,14 @@ import (
"k8s.io/klog/v2"

"github.com/Azure/azure-service-operator/hack/generator/pkg/astmodel"
"github.com/Azure/azure-service-operator/hack/generator/pkg/codegen/pipeline"
"github.com/Azure/azure-service-operator/hack/generator/pkg/config"
)

// CodeGenerator is a generator of code
type CodeGenerator struct {
configuration *config.Configuration
pipeline []PipelineStage
}

func translatePipelineToTarget(pipeline config.GenerationPipeline) (PipelineTarget, error) {
switch pipeline {
case config.GenerationPipelineAzure:
return ARMTarget, nil
case config.GenerationPipelineCrossplane:
return CrossplaneTarget, nil
default:
return PipelineTarget{}, errors.Errorf("unknown pipeline target kind %s", pipeline)
}
pipeline []pipeline.Stage
}

// NewCodeGeneratorFromConfigFile produces a new Generator with the given configuration file
Expand All @@ -41,7 +31,7 @@ func NewCodeGeneratorFromConfigFile(configurationFile string) (*CodeGenerator, e
return nil, err
}

target, err := translatePipelineToTarget(configuration.Pipeline)
target, err := pipeline.TranslatePipelineToTarget(configuration.Pipeline)
if err != nil {
return nil, err
}
Expand All @@ -50,17 +40,17 @@ func NewCodeGeneratorFromConfigFile(configurationFile string) (*CodeGenerator, e
}

// NewTargetedCodeGeneratorFromConfig produces a new code generator with the given configuration and
// only the stages appropriate for the specfied target.
// only the stages appropriate for the specified target.
func NewTargetedCodeGeneratorFromConfig(
configuration *config.Configuration, idFactory astmodel.IdentifierFactory, target PipelineTarget) (*CodeGenerator, error) {
configuration *config.Configuration, idFactory astmodel.IdentifierFactory, target pipeline.Target) (*CodeGenerator, error) {

result, err := NewCodeGeneratorFromConfig(configuration, idFactory)
if err != nil {
return nil, errors.Wrapf(err, "creating pipeline targeting %v", target)
}

// Filter stages to use only those appropriate for our target
var stages []PipelineStage
var stages []pipeline.Stage
for _, s := range result.pipeline {
if s.IsUsedFor(target) {
stages = append(stages, s)
Expand All @@ -87,104 +77,104 @@ func NewCodeGeneratorFromConfig(configuration *config.Configuration, idFactory a
return result, nil
}

func createAllPipelineStages(idFactory astmodel.IdentifierFactory, configuration *config.Configuration) []PipelineStage {
return []PipelineStage{
func createAllPipelineStages(idFactory astmodel.IdentifierFactory, configuration *config.Configuration) []pipeline.Stage {
return []pipeline.Stage{

loadSchemaIntoTypes(idFactory, configuration, defaultSchemaLoader),
pipeline.LoadSchemaIntoTypes(idFactory, configuration, pipeline.DefaultSchemaLoader),

// Import status info from Swagger:
augmentResourcesWithStatus(idFactory, configuration),
pipeline.AugmentResourcesWithStatus(idFactory, configuration),

// Reduces oneOf/allOf types from schemas to object types:
convertAllOfAndOneOfToObjects(idFactory),
pipeline.ConvertAllOfAndOneOfToObjects(idFactory),

// Flatten out any nested resources created by allOf, etc. we want to do this before naming types or things
// get named with names like Resource_Spec_Spec_Spec:
flattenResources(),
pipeline.FlattenResources(),

stripUnreferencedTypeDefinitions(),
pipeline.StripUnreferencedTypeDefinitions(),

// Name all anonymous object, enum, and validated types (required by controller-gen):
nameTypesForCRD(idFactory),
pipeline.NameTypesForCRD(idFactory),

// Apply property type rewrites from the config file
// Must come after nameTypesForCRD ('nameTypes)' and convertAllOfAndOneOfToObjects ('allof-anyof-objects') so
// Must come after NameTypesForCRD ('nameTypes)' and ConvertAllOfAndOneOfToObjects ('allof-anyof-objects') so
// that objects are all expanded
applyPropertyRewrites(configuration).
pipeline.ApplyPropertyRewrites(configuration).
RequiresPrerequisiteStages("nameTypes", "allof-anyof-objects"),

// Figure out resource owners:
determineResourceOwnership(configuration),
pipeline.DetermineResourceOwnership(configuration),

// Strip out redundant type aliases:
removeTypeAliases(),
pipeline.RemoveTypeAliases(),

// Collapse cross group references
collapseCrossGroupReferences(),
pipeline.CollapseCrossGroupReferences(),

// De-pluralize resource types
// (Must come after type aliases are resolved)
improveResourcePluralization().
pipeline.ImproveResourcePluralization().
RequiresPrerequisiteStages("removeAliases"),

stripUnreferencedTypeDefinitions(),
pipeline.StripUnreferencedTypeDefinitions(),

assertTypesCollectionValid(),
pipeline.AssertTypesCollectionValid(),

removeEmbeddedResources().UsedFor(ARMTarget), // TODO: For now only used for ARM,
pipeline.RemoveEmbeddedResources().UsedFor(pipeline.ARMTarget), // TODO: For now only used for ARM,

// Apply export filters before generating
// ARM types for resources etc:
applyExportFilters(configuration),
pipeline.ApplyExportFilters(configuration),

stripUnreferencedTypeDefinitions(),
pipeline.StripUnreferencedTypeDefinitions(),

replaceAnyTypeWithJSON(),
pipeline.ReplaceAnyTypeWithJSON(),

addCrossResourceReferences(configuration, idFactory).UsedFor(ARMTarget),
pipeline.AddCrossResourceReferences(configuration, idFactory).UsedFor(pipeline.ARMTarget),

reportOnTypesAndVersions(configuration).UsedFor(ARMTarget), // TODO: For now only used for ARM
pipeline.ReportOnTypesAndVersions(configuration).UsedFor(pipeline.ARMTarget), // TODO: For now only used for ARM

createARMTypes(idFactory).UsedFor(ARMTarget),
applyARMConversionInterface(idFactory).UsedFor(ARMTarget),
applyKubernetesResourceInterface(idFactory).UsedFor(ARMTarget),
pipeline.CreateARMTypes(idFactory).UsedFor(pipeline.ARMTarget),
pipeline.ApplyARMConversionInterface(idFactory).UsedFor(pipeline.ARMTarget),
pipeline.ApplyKubernetesResourceInterface(idFactory).UsedFor(pipeline.ARMTarget),

addCrossplaneOwnerProperties(idFactory).UsedFor(CrossplaneTarget),
addCrossplaneForProvider(idFactory).UsedFor(CrossplaneTarget),
addCrossplaneAtProvider(idFactory).UsedFor(CrossplaneTarget),
addCrossplaneEmbeddedResourceSpec(idFactory).UsedFor(CrossplaneTarget),
addCrossplaneEmbeddedResourceStatus(idFactory).UsedFor(CrossplaneTarget),
pipeline.AddCrossplaneOwnerProperties(idFactory).UsedFor(pipeline.CrossplaneTarget),
pipeline.AddCrossplaneForProvider(idFactory).UsedFor(pipeline.CrossplaneTarget),
pipeline.AddCrossplaneAtProvider(idFactory).UsedFor(pipeline.CrossplaneTarget),
pipeline.AddCrossplaneEmbeddedResourceSpec(idFactory).UsedFor(pipeline.CrossplaneTarget),
pipeline.AddCrossplaneEmbeddedResourceStatus(idFactory).UsedFor(pipeline.CrossplaneTarget),

createStorageTypes(idFactory).UsedFor(ARMTarget), // TODO: For now only used for ARM
simplifyDefinitions(),
injectJsonSerializationTests(idFactory).UsedFor(ARMTarget),
pipeline.CreateStorageTypes(idFactory).UsedFor(pipeline.ARMTarget), // TODO: For now only used for ARM
pipeline.SimplifyDefinitions(),
pipeline.InjectJsonSerializationTests(idFactory).UsedFor(pipeline.ARMTarget),

markStorageVersion(),
pipeline.MarkStorageVersion(),

// Safety checks at the end:
ensureDefinitionsDoNotUseAnyTypes(),
ensureARMTypeExistsForEveryResource().UsedFor(ARMTarget),
pipeline.EnsureDefinitionsDoNotUseAnyTypes(),
pipeline.EnsureARMTypeExistsForEveryResource().UsedFor(pipeline.ARMTarget),

deleteGeneratedCode(configuration.FullTypesOutputPath()),
pipeline.DeleteGeneratedCode(configuration.FullTypesOutputPath()),

exportPackages(configuration.FullTypesOutputPath()).
pipeline.ExportPackages(configuration.FullTypesOutputPath()).
RequiresPrerequisiteStages("deleteGenerated"),

exportControllerResourceRegistrations(configuration.FullTypesRegistrationOutputFilePath()).UsedFor(ARMTarget),
pipeline.ExportControllerResourceRegistrations(configuration.FullTypesRegistrationOutputFilePath()).UsedFor(pipeline.ARMTarget),
}
}

// Generate produces the Go code corresponding to the configured JSON schema in the given output folder
func (generator *CodeGenerator) Generate(ctx context.Context) error {
klog.V(1).Infof("Generator version: %v", combinedVersion())
klog.V(1).Infof("Generator version: %v", pipeline.CombinedVersion())

defs := make(astmodel.Types)
for i, stage := range generator.pipeline {
klog.V(0).Infof("%d/%d: %s", i+1, len(generator.pipeline), stage.description)
klog.V(0).Infof("%d/%d: %s", i+1, len(generator.pipeline), stage.Description())
// Defensive copy (in case the pipeline modifies its inputs) so that we can compare types in vs out
defsOut, err := stage.action(ctx, defs.Copy())
defsOut, err := stage.Run(ctx, defs.Copy())
if err != nil {
return errors.Wrapf(err, "failed during pipeline stage %d/%d: %s", i+1, len(generator.pipeline), stage.description)
return errors.Wrapf(err, "failed during pipeline stage %d/%d: %s", i+1, len(generator.pipeline), stage.Description())
}

defsAdded := defsOut.Except(defs)
Expand Down Expand Up @@ -216,18 +206,17 @@ func (generator *CodeGenerator) verifyPipeline() error {
stagesExpected := make(map[string][]string)

for _, stage := range generator.pipeline {
for _, prereq := range stage.prerequisites {
if _, ok := stagesSeen[prereq]; !ok {
errs = append(errs, errors.Errorf("prerequisite %q of stage %q not satisfied.", prereq, stage.id))
}
err := stage.CheckPrerequisites(stagesSeen)
if err != nil {
errs = append(errs, err)
}

for _, postreq := range stage.postrequisites {
stagesExpected[postreq] = append(stagesExpected[postreq], stage.id)
for _, postreq := range stage.Postrequisites() {
stagesExpected[postreq] = append(stagesExpected[postreq], stage.Id())
}

stagesSeen[stage.id] = struct{}{}
delete(stagesExpected, stage.id)
stagesSeen[stage.Id()] = struct{}{}
delete(stagesExpected, stage.Id())
}

for required, requiredBy := range stagesExpected {
Expand All @@ -248,30 +237,30 @@ func (generator *CodeGenerator) RemoveStages(stageIds ...string) {
stagesToRemove[s] = false
}

var pipeline []PipelineStage
var stages []pipeline.Stage

for _, stage := range generator.pipeline {
if _, ok := stagesToRemove[stage.id]; ok {
stagesToRemove[stage.id] = true
if _, ok := stagesToRemove[stage.Id()]; ok {
stagesToRemove[stage.Id()] = true
continue
}

pipeline = append(pipeline, stage)
stages = append(stages, stage)
}

for stage, removed := range stagesToRemove {
if !removed {
panic(fmt.Sprintf("Expected to remove stage %s from pipeline, but it wasn't found.", stage))
panic(fmt.Sprintf("Expected to remove stage %s from stages, but it wasn't found.", stage))
}
}

generator.pipeline = pipeline
generator.pipeline = stages
}

// ReplaceStage replaces all uses of an existing stage with another one.
// Only available for test builds.
// Will panic if the existing stage is not found.
func (generator *CodeGenerator) ReplaceStage(existingStage string, stage PipelineStage) {
func (generator *CodeGenerator) ReplaceStage(existingStage string, stage pipeline.Stage) {
replaced := false
for i, s := range generator.pipeline {
if s.HasId(existingStage) {
Expand All @@ -288,12 +277,12 @@ func (generator *CodeGenerator) ReplaceStage(existingStage string, stage Pipelin
// InjectStageAfter injects a new stage immediately after the first occurrence of an existing stage
// Only available for test builds.
// Will panic if the existing stage is not found.
func (generator *CodeGenerator) InjectStageAfter(existingStage string, stage PipelineStage) {
func (generator *CodeGenerator) InjectStageAfter(existingStage string, stage pipeline.Stage) {
injected := false

for i, s := range generator.pipeline {
if s.HasId(existingStage) {
var p []PipelineStage
var p []pipeline.Stage
p = append(p, generator.pipeline[:i+1]...)
p = append(p, stage)
p = append(p, generator.pipeline[i+1:]...)
Expand All @@ -304,7 +293,7 @@ func (generator *CodeGenerator) InjectStageAfter(existingStage string, stage Pip
}

if !injected {
panic(fmt.Sprintf("Expected to inject stage %s but %s wasn't found", stage.id, existingStage))
panic(fmt.Sprintf("Expected to inject stage %s but %s wasn't found", stage.Id(), existingStage))
}
}

Expand Down
Loading

0 comments on commit f549305

Please sign in to comment.