From 1bd5ffc98c88fdde6d11383a10e53199a9d1e33a Mon Sep 17 00:00:00 2001 From: Anthony Martin Date: Tue, 16 Mar 2021 18:54:00 -0400 Subject: [PATCH] Add test framework to evaluate a template containing expressions (#1875) * Add framework to evaluate a template * Add more evaluation tests --- .../Bicep.Core.IntegrationTests.csproj | 1 + .../EvaluationTests.cs | 262 ++++++++++++++++++ .../TemplateEvaluator.cs | 189 +++++++++++++ 3 files changed, 452 insertions(+) create mode 100644 src/Bicep.Core.IntegrationTests/EvaluationTests.cs create mode 100644 src/Bicep.Core.IntegrationTests/TemplateEvaluator.cs diff --git a/src/Bicep.Core.IntegrationTests/Bicep.Core.IntegrationTests.csproj b/src/Bicep.Core.IntegrationTests/Bicep.Core.IntegrationTests.csproj index f73110a1a1f..115cadad25f 100644 --- a/src/Bicep.Core.IntegrationTests/Bicep.Core.IntegrationTests.csproj +++ b/src/Bicep.Core.IntegrationTests/Bicep.Core.IntegrationTests.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Bicep.Core.IntegrationTests/EvaluationTests.cs b/src/Bicep.Core.IntegrationTests/EvaluationTests.cs new file mode 100644 index 00000000000..b0e4b31c149 --- /dev/null +++ b/src/Bicep.Core.IntegrationTests/EvaluationTests.cs @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System; +using Bicep.Core.UnitTests.Assertions; +using Bicep.Core.UnitTests.Utils; +using FluentAssertions; +using FluentAssertions.Execution; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json.Linq; + +namespace Bicep.Core.IntegrationTests +{ + [TestClass] + public class EvaluationTests + { + [TestMethod] + public void Basic_arithmetic_expressions_are_evaluated_successfully() + { + var (template, _, _) = CompilationHelper.Compile(@" +var sum = 1 + 3 +var mult = sum * 5 +var modulo = mult % 7 +var div = modulo / 11 +var sub = div - 13 + +output sum int = sum +output mult int = mult +output modulo int = modulo +output div int = div +output sub int = sub +"); + + using (new AssertionScope()) + { + var evaluated = TemplateEvaluator.Evaluate(template); + + evaluated.Should().HaveValueAtPath("$.outputs['sum'].value", 4); + evaluated.Should().HaveValueAtPath("$.outputs['mult'].value", 20); + evaluated.Should().HaveValueAtPath("$.outputs['modulo'].value", 6); + evaluated.Should().HaveValueAtPath("$.outputs['div'].value", 0); + evaluated.Should().HaveValueAtPath("$.outputs['sub'].value", -13); + } + } + + [TestMethod] + public void String_expressions_are_evaluated_successfully() + { + var (template, _, _) = CompilationHelper.Compile(@" +var bool = false +var int = 12948 +var str = 'hello!' +var obj = { + a: 'b' + '!c': 2 +} +var arr = [ + true + 2893 + 'abc' +] +var multiline = ''' +these escapes + are not + evaluted: +\r\n\t\\\'\${} +''' + +output literal string = str +output interp string = '>${bool}<>${int}<>${str}<>${obj}<>${arr}<' +output escapes string = '\r\n\t\\\'\${}' +output multiline string = multiline +"); + + using (new AssertionScope()) + { + var evaluated = TemplateEvaluator.Evaluate(template); + + evaluated.Should().HaveValueAtPath("$.outputs['literal'].value", "hello!"); + evaluated.Should().HaveValueAtPath("$.outputs['interp'].value", ">False<>12948<>hello!<>{'a':'b','!c':2}<>[true,2893,'abc']<"); + evaluated.Should().HaveValueAtPath("$.outputs['escapes'].value", "\r\n\t\\'${}"); + evaluated.Should().HaveValueAtPath("$.outputs['multiline'].value", "these escapes\n are not\n evaluted:\n\\r\\n\\t\\\\\\'\\${}\n"); + } + } + + [TestMethod] + public void ResourceId_expressions_are_evaluated_successfully() + { + var (template, _, _) = CompilationHelper.Compile(@" +param parentName string +param childName string + +resource existing1 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' +} + +resource existing2 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: resourceGroup('customRg') +} + +resource existing3 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: resourceGroup('2e518a80-f860-4467-a00e-d81aaf1ff42e', 'customRg') +} + +resource existing4 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: tenant() +} + +resource existing5 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: subscription() +} + +resource existing6 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: subscription('2e518a80-f860-4467-a00e-d81aaf1ff42e') +} + +resource existing7 'My.Rp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: managementGroup('2e518a80-f860-4467-a00e-d81aaf1ff42e') +} + +resource existing8 'My.ExtensionRp/parent/child@2020-01-01' existing = { + name: '${parentName}/${childName}' + scope: existing4 +} + +output resource1Id string = existing1.id +output resource2Id string = existing2.id +output resource3Id string = existing3.id +output resource4Id string = existing4.id +output resource5Id string = existing5.id +output resource6Id string = existing6.id +output resource7Id string = existing7.id +output resource8Id string = existing8.id + +output resource1Name string = existing1.name +output resource1ApiVersion string = existing1.apiVersion +output resource1Type string = existing1.type +"); + + using (new AssertionScope()) + { + var testSubscriptionId = "87d64d6d-6d17-4ad7-b507-16d9bc498781"; + var testRgName = "testRg"; + var evaluated = TemplateEvaluator.Evaluate(template, config => config with { + SubscriptionId = testSubscriptionId, + ResourceGroup = testRgName, + Parameters = new() { + ["parentName"] = "myParent", + ["childName"] = "myChild", + } + }); + + evaluated.Should().HaveValueAtPath("$.outputs['resource1Id'].value", $"/subscriptions/{testSubscriptionId}/resourceGroups/{testRgName}/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource2Id'].value", $"/subscriptions/{testSubscriptionId}/resourceGroups/customRg/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource3Id'].value", $"/subscriptions/2e518a80-f860-4467-a00e-d81aaf1ff42e/resourceGroups/customRg/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource4Id'].value", $"/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource5Id'].value", $"/subscriptions/{testSubscriptionId}/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource6Id'].value", $"/subscriptions/2e518a80-f860-4467-a00e-d81aaf1ff42e/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource7Id'].value", $"/providers/Microsoft.Management/managementGroups/2e518a80-f860-4467-a00e-d81aaf1ff42e/providers/My.Rp/parent/myParent/child/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource8Id'].value", $"/providers/My.Rp/parent/myParent/child/myChild/providers/My.ExtensionRp/parent/myParent/child/myChild"); + + evaluated.Should().HaveValueAtPath("$.outputs['resource1Name'].value", "myParent/myChild"); + evaluated.Should().HaveValueAtPath("$.outputs['resource1ApiVersion'].value", "2020-01-01"); + evaluated.Should().HaveValueAtPath("$.outputs['resource1Type'].value", "My.Rp/parent/child"); + } + } + + [TestMethod] + public void Comparisons_are_evaluated_correctly() + { + var (template, _, _) = CompilationHelper.Compile(@" +output less bool = 123 < 456 +output lessOrEquals bool = 123 <= 456 +output greater bool = 123 > 456 +output greaterOrEquals bool = 123 >= 456 +output equals bool = 123 == 456 +output not bool = !true +output and bool = true && false +output or bool = true || false +output coalesce int = null ?? 123 +"); + + using (new AssertionScope()) + { + var evaluated = TemplateEvaluator.Evaluate(template); + + evaluated.Should().HaveValueAtPath("$.outputs['less'].value", true); + evaluated.Should().HaveValueAtPath("$.outputs['lessOrEquals'].value", true); + evaluated.Should().HaveValueAtPath("$.outputs['greater'].value", false); + evaluated.Should().HaveValueAtPath("$.outputs['greaterOrEquals'].value", false); + evaluated.Should().HaveValueAtPath("$.outputs['equals'].value", false); + evaluated.Should().HaveValueAtPath("$.outputs['not'].value", false); + evaluated.Should().HaveValueAtPath("$.outputs['and'].value", false); + evaluated.Should().HaveValueAtPath("$.outputs['or'].value", true); + evaluated.Should().HaveValueAtPath("$.outputs['coalesce'].value", 123); + } + } + + [TestMethod] + public void Resource_property_access_works() + { + var (template, _, _) = CompilationHelper.Compile(@" +param abcVal string + +resource testRes 'My.Rp/res1@2020-01-01' = { + name: 'testRes' + properties: { + abc: abcVal + } +} + +output abcVal string = testRes.properties.abc +"); + + using (new AssertionScope()) + { + var evaluated = TemplateEvaluator.Evaluate(template, config => config with { + Parameters = new() { + ["abcVal"] = "test!!!", + }, + }); + + evaluated.Should().HaveValueAtPath("$.outputs['abcVal'].value", "test!!!"); + } + } + + [TestMethod] + public void Existing_resource_property_access_works() + { + var (template, _, _) = CompilationHelper.Compile(@" +resource testRes 'My.Rp/res1@2020-01-01' existing = { + name: 'testRes' +} + +output abcVal string = testRes.properties.abc +"); + + using (new AssertionScope()) + { + var evaluated = TemplateEvaluator.Evaluate(template, config => config with { + OnReferenceFunc = (resourceId, apiVersion, fullBody) => { + if (resourceId == $"/subscriptions/{config.SubscriptionId}/resourceGroups/{config.ResourceGroup}/providers/My.Rp/res1/testRes" && apiVersion == "2020-01-01" && !fullBody) + { + return new JObject { + ["abc"] = "test!!!", + }; + } + + throw new NotImplementedException(); + }, + }); + + evaluated.Should().HaveValueAtPath("$.outputs['abcVal'].value", "test!!!"); + } + } + } +} \ No newline at end of file diff --git a/src/Bicep.Core.IntegrationTests/TemplateEvaluator.cs b/src/Bicep.Core.IntegrationTests/TemplateEvaluator.cs new file mode 100644 index 00000000000..58102a8e9eb --- /dev/null +++ b/src/Bicep.Core.IntegrationTests/TemplateEvaluator.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. +using System.Collections.Generic; +using System; +using Newtonsoft.Json.Linq; +using Azure.Deployments.Templates.Engines; +using Azure.Deployments.Templates.Configuration; +using Azure.Deployments.Core.Collections; +using Azure.Deployments.Core.Instrumentation; +using Azure.Deployments.Templates.Schema; +using Azure.Deployments.Core.Extensions; +using Azure.Deployments.Expression.Engines; +using System.Linq; +using Azure.Deployments.Expression.Expressions; +using Azure.Deployments.Core.ErrorResponses; +using System.Collections.Immutable; + +namespace Bicep.Core.IntegrationTests +{ + public class TemplateEvaluator + { + const string TestTenantId = "d4c73686-f7cd-458e-b377-67adcd46b624"; + const string TestManagementGroupName = "3fc9f36e-8699-43af-b038-1c103980942f"; + const string TestSubscriptionId = "f91a30fd-f403-4999-ae9f-ec37a6d81e13"; + const string TestResourceGroupName = "testResourceGroup"; + const string TestLocation = "West US"; + + public delegate JToken OnListDelegate(string functionName, string resourceId, string apiVersion, JToken? body); + + public delegate JToken OnReferenceDelegate(string resourceId, string apiVersion, bool fullBody); + + public record EvaluationConfiguration( + string TenantId, + string ManagementGroup, + string SubscriptionId, + string ResourceGroup, + string RgLocation, + Dictionary Parameters, + OnListDelegate? OnListFunc, + OnReferenceDelegate? OnReferenceFunc) + { + public static EvaluationConfiguration Default = new( + TestTenantId, + TestManagementGroupName, + TestSubscriptionId, + TestResourceGroupName, + TestLocation, + new(), + null, + null + ); + } + + private static string GetResourceId(string scopeString, TemplateResource resource) + { + var typeSegments = resource.Type.Value.Split('/'); + var nameSegments = resource.Name.Value.Split('/'); + + var types = new [] { typeSegments.First() } + .Concat(typeSegments.Skip(1).Zip(nameSegments, (type, name) => $"{type}/{name}")); + + return $"{scopeString}providers/{string.Join('/', types)}"; + } + + private static void ProcessTemplateLanguageExpressions(Template template, EvaluationConfiguration config, TemplateDeploymentScope deploymentScope) + { + var scopeString = deploymentScope switch { + TemplateDeploymentScope.Tenant => "/", + TemplateDeploymentScope.ManagementGroup => $"/providers/Microsoft.Management/managementGroups/{config.ManagementGroup}/", + TemplateDeploymentScope.Subscription => $"/subscriptions/{config.SubscriptionId}/", + TemplateDeploymentScope.ResourceGroup => $"/subscriptions/{config.SubscriptionId}/resourceGroups/{config.ResourceGroup}/", + _ => throw new InvalidOperationException(), + }; + + var resourceLookup = template.Resources.ToOrdinalInsensitiveDictionary(x => GetResourceId(scopeString, x)); + + var evaluationContext = TemplateEngine.GetExpressionEvaluationContext(template); + var defaultEvaluateFunction = evaluationContext.EvaluateFunction; + evaluationContext.EvaluateFunction = (FunctionExpression functionExpression, JToken[] parameters, TemplateErrorAdditionalInfo additionalInfo) => + { + if (functionExpression.Function.StartsWithOrdinalInsensitively("list") && config.OnListFunc is not null) + { + return config.OnListFunc( + functionExpression.Function, + parameters[0].ToString(), + parameters[1].ToString(), + parameters.Length > 2 ? parameters[2] : null); + } + + if (functionExpression.Function.EqualsOrdinalInsensitively("reference")) + { + var resourceId = parameters[0].ToString(); + var apiVersion = parameters.Length > 1 ? parameters[1].ToString() : null; + var fullBody = parameters.Length > 2 ? parameters[2].ToString().EqualsOrdinalInsensitively("Full") : false; + + if (resourceLookup.TryGetValue(resourceId, out var foundResource) && + (apiVersion is null || StringComparer.OrdinalIgnoreCase.Equals(apiVersion, foundResource.ApiVersion.Value))) + { + return fullBody ? foundResource.ToJToken() : foundResource.Properties.ToJToken(); + } + + if (apiVersion is not null && config.OnReferenceFunc is not null) + { + return config.OnReferenceFunc(resourceId, apiVersion, fullBody); + } + } + + var value = defaultEvaluateFunction(functionExpression, parameters, additionalInfo); + return value; + }; + + for (int i = 0; i < template.Resources.Length; i++) + { + var resource = template.Resources[i]; + + resource.Properties.Value = ExpressionsEngine.EvaluateLanguageExpressionsRecursive( + root: resource.Properties.Value, + evaluationContext: evaluationContext); + } + + if (template.Outputs is not null && template.Outputs.Count > 0) + { + foreach (var outputKey in template.Outputs.Keys.ToList()) + { + template.Outputs[outputKey].Value.Value = ExpressionsEngine.EvaluateLanguageExpressionsRecursive( + root: template.Outputs[outputKey].Value.Value, + evaluationContext: evaluationContext); + } + } + } + + public static JToken Evaluate(JToken? templateJtoken, Func? configurationBuilder = null) + { + var configuration = EvaluationConfiguration.Default; + + if (configurationBuilder is not null) + { + configuration = configurationBuilder(configuration); + } + + return EvaluateTemplate(templateJtoken, configuration); + } + + private static JToken EvaluateTemplate(JToken? templateJtoken, EvaluationConfiguration config) + { + DeploymentsInterop.Initialize(); + + templateJtoken = templateJtoken ?? throw new ArgumentNullException(nameof(templateJtoken)); + + var deploymentScope = templateJtoken["$schema"]?.ToString() switch { + "https://schema.management.azure.com/schemas/2019-08-01/tenantDeploymentTemplate.json#" => TemplateDeploymentScope.Tenant, + "https://schema.management.azure.com/schemas/2019-08-01/managementGroupDeploymentTemplate.json#" => TemplateDeploymentScope.ManagementGroup, + "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#" => TemplateDeploymentScope.Subscription, + "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#" => TemplateDeploymentScope.ResourceGroup, + _ => throw new InvalidOperationException(), + }; + + var metadata = new InsensitiveDictionary(); + if (deploymentScope == TemplateDeploymentScope.Subscription || deploymentScope == TemplateDeploymentScope.ResourceGroup) + { + metadata["subscription"] = new JObject { + ["id"] = $"/subscriptions/{config.SubscriptionId}", + ["subscriptionId"] = config.SubscriptionId, + ["tenantId"] = config.TenantId, + }; + } + if (deploymentScope == TemplateDeploymentScope.ResourceGroup) + { + metadata["resourceGroup"] = new JObject { + ["id"] = $"/subscriptions/{config.SubscriptionId}/resourceGroups/{config.ResourceGroup}", + ["location"] = config.RgLocation, + }; + }; + + var template = TemplateEngine.ParseTemplate(templateJtoken.ToString()); + + TemplateEngine.ValidateTemplate(template, "2020-06-01", deploymentScope); + TemplateEngine.ParameterizeTemplate(template, new InsensitiveDictionary(config.Parameters), metadata, new InsensitiveDictionary()); + + TemplateEngine.ProcessTemplateLanguageExpressions(template, "2020-06-01"); + + ProcessTemplateLanguageExpressions(template, config, deploymentScope); + + TemplateEngine.ValidateProcessedTemplate(template, "2020-06-01", deploymentScope); + + return template.ToJToken(); + } + } +} \ No newline at end of file