diff --git a/examples/jenkins/application-template.json b/examples/jenkins/application-template.json index 3fc40289cade..6b1f148ed89f 100644 --- a/examples/jenkins/application-template.json +++ b/examples/jenkins/application-template.json @@ -115,10 +115,10 @@ } ], "resources": { - "limits": { - "memory": "${MEMORY_LIMIT}" - } - }, + "limits": { + "memory": "${MEMORY_LIMIT}" + } + }, "terminationMessagePath": "/dev/termination-log", "imagePullPolicy": "IfNotPresent", "securityContext": { @@ -134,7 +134,6 @@ }, "status": {} }, - { "kind": "Service", "apiVersion": "v1", @@ -297,10 +296,10 @@ } ], "resources": { - "limits": { - "memory": "${MEMORY_LIMIT}" - } - }, + "limits": { + "memory": "${MEMORY_LIMIT}" + } + }, "terminationMessagePath": "/dev/termination-log", "imagePullPolicy": "IfNotPresent", "securityContext": { diff --git a/pkg/template/registry/rest.go b/pkg/template/registry/rest.go index 0ab751d9004f..01c704ce9876 100644 --- a/pkg/template/registry/rest.go +++ b/pkg/template/registry/rest.go @@ -47,10 +47,15 @@ func (s *REST) Create(ctx kapi.Context, obj runtime.Object) (runtime.Object, err "expression": generator.NewExpressionValueGenerator(rand.New(rand.NewSource(time.Now().UnixNano()))), } processor := template.NewProcessor(generators) - if errs := processor.Process(tpl); len(errs) > 0 { + errs, err := processor.Process(tpl) + if len(errs) > 0 { glog.V(1).Infof(errs.ToAggregate().Error()) return nil, errors.NewInvalid(api.Kind("Template"), tpl.Name, errs) } + if err != nil { + glog.V(1).Infof(err.Error()) + return nil, errors.NewInternalError(err) + } // we know that we get back runtime.Unstructured objects from the Process call. We need to encode those // objects using the unstructured codec BEFORE the REST layers gets its shot at encoding to avoid a layered diff --git a/pkg/template/template.go b/pkg/template/template.go index 68b081354416..5982d13b1d49 100644 --- a/pkg/template/template.go +++ b/pkg/template/template.go @@ -1,10 +1,13 @@ package template import ( + // "encoding/json" + "errors" "fmt" "regexp" "strings" + kapi "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/meta" "k8s.io/kubernetes/pkg/runtime" "k8s.io/kubernetes/pkg/util/validation/field" @@ -12,10 +15,19 @@ import ( "github.com/openshift/origin/pkg/template/api" . "github.com/openshift/origin/pkg/template/generator" "github.com/openshift/origin/pkg/util" - "github.com/openshift/origin/pkg/util/stringreplace" ) -var parameterExp = regexp.MustCompile(`\$\{([a-zA-Z0-9\_]+)\}`) +// match ${KEY}, KEY will be grouped +var stringParameterExp = regexp.MustCompile(`\$\{([a-zA-Z0-9\_]+?)\}`) + +// match ${{KEY}}, KEY will be grouped +var nonStringParameterExp = regexp.MustCompile(`\$\{\{([a-zA-Z0-9\_]+?)\}\}`) + +// any quoted string +var fieldExp = regexp.MustCompile(`".*?"`) + +// non-greedy match for "blah blah ${{KEY}} blah blah", including the quotes. +//var fieldWithNonStringParameterExp = regexp.MustCompile(`\".*\$\{\{[a-zA-Z0-9\_]+?\}\}.*\"`) // Processor process the Template into the List with substituted parameters type Processor struct { @@ -30,24 +42,13 @@ func NewProcessor(generators map[string]Generator) *Processor { // Process transforms Template object into List object. It generates // Parameter values using the defined set of generators first, and then it // substitutes all Parameter expression occurrences with their corresponding -// values (currently in the containers' Environment variables only). -func (p *Processor) Process(template *api.Template) field.ErrorList { +// values. +func (p *Processor) Process(template *api.Template) (field.ErrorList, error) { templateErrors := field.ErrorList{} - if fieldError := p.GenerateParameterValues(template); fieldError != nil { - return append(templateErrors, fieldError) - } - - // Place parameters into a map for efficient lookup - paramMap := make(map[string]api.Parameter) - for _, param := range template.Parameters { - paramMap[param.Name] = param - } - - // Perform parameter substitution on the template's user message. This can be used to - // instruct a user on next steps for the template. - template.Message = p.EvaluateParameterSubstitution(paramMap, template.Message) - + // We start with a list of runtime.Unknown objects in the template, so first we need to + // Decode them to runtime.Unstructured so we can manipulate the set of Labels and strip + // the Namespace. itemPath := field.NewPath("item") for i, item := range template.Objects { idxPath := itemPath.Index(i) @@ -60,24 +61,59 @@ func (p *Processor) Process(template *api.Template) field.ErrorList { } item = decodedObj } - - newItem, err := p.SubstituteParameters(paramMap, item) - if err != nil { - templateErrors = append(templateErrors, field.Invalid(idxPath.Child("parameters"), template.Parameters, err.Error())) - } // If an object definition's metadata includes a namespace field, the field will be stripped out of // the definition during template instantiation. This is necessary because all objects created during // instantiation are placed into the target namespace, so it would be invalid for the object to declare - //a different namespace. - stripNamespace(newItem) - if err := util.AddObjectLabels(newItem, template.ObjectLabels); err != nil { + // a different namespace. + stripNamespace(item) + if err := util.AddObjectLabels(item, template.ObjectLabels); err != nil { templateErrors = append(templateErrors, field.Invalid(idxPath.Child("labels"), template.ObjectLabels, fmt.Sprintf("label could not be applied: %v", err))) } - template.Objects[i] = newItem + template.Objects[i] = item + } + if fieldError := p.GenerateParameterValues(template); fieldError != nil { + templateErrors = append(templateErrors, fieldError) + } + + // accrued errors from processing the template objects and parameters + if len(templateErrors) != 0 { + return templateErrors, nil + } + + // Place parameters into a map for efficient lookup + paramMap := make(map[string]api.Parameter) + for _, param := range template.Parameters { + paramMap[param.Name] = param + } + + // Turn the template object into json so we can do search/replace on it + // to substitute parameter values. + serializer, found := kapi.Codecs.SerializerForMediaType("application/json", nil) + if !found { + return templateErrors, errors.New("Could not load json serializer") } - return templateErrors + templateBytes, err := runtime.Encode(serializer, template) + if err != nil { + return templateErrors, err + } + templateString := string(templateBytes) + + // consider we start with a field like "${PARAM1}${{PARAM2}" + // if we substitute and strip quotes first, we're left with + // ${PARAM1}VALUE2 and then when we search for ${{KEY}} parameters + // to replace, we won't find any because the value is not inside quotes anymore. + // So instead we must do the string-parameter substitution first, so we have + // "VALUE1${{PARAM2}}" which we can then substitute into VALUE1VALUE2. + templateString = p.EvaluateParameterSubstitution(paramMap, templateString, true) + templateString = p.EvaluateParameterSubstitution(paramMap, templateString, false) + + // Now that the json is properly substituted and de-quoted where needed for non-string + // field values, decode the json back into the template object. This will leave us + // with runtime.Unstructured json structs in the Object list again. + err = runtime.DecodeInto(kapi.Codecs.UniversalDecoder(), []byte(templateString), template) + return templateErrors, err } func stripNamespace(obj runtime.Object) { @@ -126,30 +162,40 @@ func GetParameterByName(t *api.Template, name string) *api.Parameter { // EvaluateParameterSubstitution replaces escaped parameters in a string with values from the // provided map. -func (p *Processor) EvaluateParameterSubstitution(params map[string]api.Parameter, in string) string { - for _, match := range parameterExp.FindAllStringSubmatch(in, -1) { - if len(match) > 1 { - if paramValue, found := params[match[1]]; found { - in = strings.Replace(in, match[0], paramValue.Value, 1) +func (p *Processor) EvaluateParameterSubstitution(params map[string]api.Parameter, in string, stringParameters bool) string { + + // find quoted blocks + for _, fieldValue := range fieldExp.FindAllStringSubmatch(in, -1) { + origFieldValue := fieldValue[0] + // find ${{KEY}} or ${KEY} entries in the string + parameterExp := stringParameterExp + if !stringParameters { + parameterExp = nonStringParameterExp + } + subbed := false + for _, parameterRef := range parameterExp.FindAllStringSubmatch(fieldValue[0], -1) { + if len(parameterRef) > 1 { + // parameterRef[0] contains a field with a parameter reference like "SOME ${PARAM_KEY}" (including the quotes) + // parameterRef[1] contains PARAM_KEY + if paramValue, found := params[parameterRef[1]]; found { + // fieldValue[0] will now contain "SOME PARAM_VALUE" (including the quotes) + fieldValue[0] = strings.Replace(fieldValue[0], parameterRef[0], paramValue.Value, 1) + subbed = true + } + } + } + if subbed { + newFieldValue := fieldValue[0] + if !stringParameters { + // strip quotes from either end of the string if we matched a ${{KEY}} type parameter + newFieldValue = fieldValue[0][1 : len(fieldValue[0])-1] } + in = strings.Replace(in, origFieldValue, newFieldValue, -1) } } return in } -// SubstituteParameters loops over all values defined in structured -// and unstructured types that are children of item. -// -// Example of Parameter expression: -// - ${PARAMETER_NAME} -// -func (p *Processor) SubstituteParameters(params map[string]api.Parameter, item runtime.Object) (runtime.Object, error) { - stringreplace.VisitObjectStrings(item, func(in string) string { - return p.EvaluateParameterSubstitution(params, in) - }) - return item, nil -} - // GenerateParameterValues generates Value for each Parameter of the given // Template that has Generate field specified where Value is not already // supplied. diff --git a/pkg/template/template_test.go b/pkg/template/template_test.go index 0e6475e1e58f..710564ca681f 100644 --- a/pkg/template/template_test.go +++ b/pkg/template/template_test.go @@ -177,17 +177,22 @@ func TestParameterGenerators(t *testing.T) { } } -func TestProcessValueEscape(t *testing.T) { +func TestProcessValue(t *testing.T) { var template api.Template if err := runtime.DecodeInto(kapi.Codecs.UniversalDecoder(), []byte(`{ "kind":"Template", "apiVersion":"v1", "objects": [ { - "kind": "Service", "apiVersion": "v1${VALUE}", + "kind": "Service", "apiVersion": "v${VALUE}", "metadata": { "labels": { "key1": "${VALUE}", - "key2": "$${VALUE}" + "key2": "$${VALUE}", + "s1_s1": "${STRING_1}_${STRING_1}", + "s1_s2": "${STRING_1}_${STRING_2}", + "i1i1": "${{INT_1}}${{INT_1}}", + "i1i2": "${{INT_1}}${{INT_2}}", + "i1i2_mixed": "${INT_1}${{INT_2}}" } } } @@ -195,7 +200,6 @@ func TestProcessValueEscape(t *testing.T) { }`), &template); err != nil { t.Fatalf("unexpected error: %v", err) } - generators := map[string]generator.Generator{ "expression": generator.NewExpressionValueGenerator(rand.New(rand.NewSource(1337))), } @@ -203,19 +207,27 @@ func TestProcessValueEscape(t *testing.T) { // Define custom parameter for the transformation: AddParameter(&template, makeParameter("VALUE", "1", "", false)) + AddParameter(&template, makeParameter("STRING_1", "string1", "", false)) + AddParameter(&template, makeParameter("STRING_2", "string2", "", false)) + AddParameter(&template, makeParameter("INT_1", "1", "", false)) + AddParameter(&template, makeParameter("INT_2", "2", "", false)) // Transform the template config into the result config - errs := processor.Process(&template) + errs, err := processor.Process(&template) if len(errs) > 0 { t.Fatalf("unexpected error: %v", errs) } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } result, err := runtime.Encode(kapi.Codecs.LegacyCodec(v1.SchemeGroupVersion), &template) if err != nil { t.Fatalf("unexpected error during encoding Config: %#v", err) } - expect := `{"kind":"Template","apiVersion":"v1","metadata":{"creationTimestamp":null},"objects":[{"apiVersion":"v11","kind":"Service","metadata":{"labels":{"key1":"1","key2":"$1"}}}],"parameters":[{"name":"VALUE","value":"1"}]}` + expect := `{"kind":"Template","apiVersion":"v1","metadata":{"creationTimestamp":null},"objects":[{"apiVersion":"v1","kind":"Service","metadata":{"labels":{"i1i1":11,"i1i2":12,"i1i2_mixed":12,"key1":"1","key2":"$1","s1_s1":"string1_string1","s1_s2":"string1_string2"}}}],"parameters":[{"name":"VALUE","value":"1"},{"name":"STRING_1","value":"string1"},{"name":"STRING_2","value":"string2"},{"name":"INT_1","value":"1"},{"name":"INT_2","value":"2"}]}` stringResult := strings.TrimSpace(string(result)) if expect != stringResult { + //t.Errorf("unexpected output, expected: \n%s\nGot:\n%s\n", expect, stringResult) t.Errorf("unexpected output: %s", diff.StringDiff(expect, stringResult)) } } @@ -233,8 +245,10 @@ func TestEvaluateLabels(t *testing.T) { "kind":"Template", "apiVersion":"v1", "objects": [ { - "kind": "Service", "apiVersion": "v1", - "metadata": {"labels": {"key1": "v1", "key2": "v2"} } + "apiVersion": "v1", "kind": "Service", + "metadata": { + "labels": {"key1": "v1", "key2": "v2"} + } } ] }`, @@ -242,8 +256,10 @@ func TestEvaluateLabels(t *testing.T) { "kind":"Template","apiVersion":"v1","metadata":{"creationTimestamp":null}, "objects":[ { - "apiVersion":"v1","kind":"Service","metadata":{ - "labels":{"key1":"v1","key2":"v2"}} + "apiVersion":"v1","kind":"Service", + "metadata":{ + "labels":{"key1":"v1","key2":"v2"} + } } ] }`, @@ -253,8 +269,10 @@ func TestEvaluateLabels(t *testing.T) { "kind":"Template", "apiVersion":"v1", "objects": [ { - "kind": "Service", "apiVersion": "v1", - "metadata": {"labels": {"key1": "v1", "key2": "v2"} } + "apiVersion": "v1", "kind": "Service", + "metadata": { + "labels": {"key1": "v1", "key2": "v2"} + } } ] }`, @@ -262,25 +280,23 @@ func TestEvaluateLabels(t *testing.T) { "kind":"Template","apiVersion":"v1","metadata":{"creationTimestamp":null}, "objects":[ { - "apiVersion":"v1","kind":"Service","metadata":{ - "labels":{"key1":"v1","key2":"v2","key3":"v3"}} + "apiVersion":"v1","kind":"Service", + "metadata":{ + "labels":{"key1":"v1","key2":"v2","key3":"v3"} + } } ], "labels":{"key3":"v3"} }`, Labels: map[string]string{"key3": "v3"}, }, - "when the root object has labels and metadata": { + "root object has no labels and metadata": { Input: `{ "kind":"Template", "apiVersion":"v1", "objects": [ { - "kind": "Service", "apiVersion": "v1", - "metadata": {}, - "labels": { - "key1": "v1", - "key2": "v2" - } + "apiVersion": "v1", "kind": "Service", + "metadata": {} } ] }`, @@ -289,21 +305,24 @@ func TestEvaluateLabels(t *testing.T) { "objects":[ { "apiVersion":"v1","kind":"Service", - "labels":{"key1":"v1","key2":"v2"}, - "metadata":{"labels":{"key3":"v3"}} + "metadata":{ + "labels":{"key3":"v3"} + } } ], "labels":{"key3":"v3"} }`, Labels: map[string]string{"key3": "v3"}, }, - "overwrites label": { + "overwrites existing label": { Input: `{ "kind":"Template", "apiVersion":"v1", "objects": [ { - "kind": "Service", "apiVersion": "v1", - "metadata": {"labels": {"key1": "v1", "key2": "v2"} } + "apiVersion": "v1", "kind": "Service", + "metadata": { + "labels": {"key1": "v1", "key2": "v2"} + } } ] }`, @@ -311,8 +330,10 @@ func TestEvaluateLabels(t *testing.T) { "kind":"Template","apiVersion":"v1","metadata":{"creationTimestamp":null}, "objects":[ { - "apiVersion":"v1","kind":"Service","metadata":{ - "labels":{"key1":"v1","key2":"v3"}} + "apiVersion":"v1","kind":"Service", + "metadata":{ + "labels":{"key1":"v1","key2":"v3"} + } } ], "labels":{"key2":"v3"} @@ -336,11 +357,16 @@ func TestEvaluateLabels(t *testing.T) { template.ObjectLabels = testCase.Labels // Transform the template config into the result config - errs := processor.Process(&template) + errs, err := processor.Process(&template) if len(errs) > 0 { t.Errorf("%s: unexpected error: %v", k, errs) continue } + if err != nil { + t.Errorf("unexpected error: %v", err) + continue + } + result, err := runtime.Encode(kapi.Codecs.LegacyCodec(v1.SchemeGroupVersion), &template) if err != nil { t.Errorf("%s: unexpected error: %v", k, err) @@ -377,10 +403,13 @@ func TestProcessTemplateParameters(t *testing.T) { AddParameter(&template, makeParameter("CUSTOM_PARAM1", "1", "", false)) // Transform the template config into the result config - errs := processor.Process(&template) + errs, err := processor.Process(&template) if len(errs) > 0 { t.Fatalf("unexpected error: %v", errs) } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } result, err := runtime.Encode(kapi.Codecs.LegacyCodec(v1.SchemeGroupVersion), &template) if err != nil { t.Fatalf("unexpected error during encoding Config: %#v", err)