Add 'localDeploy' experimental feature, base extension nuget library (#…

…14231)

* Add wiring for "localDeploy" experimental feature. The feature doesn't
do anything yet as of this PR.
* Add "Bicep.Local.Extension" nuget package to contain logic for C#
extension authoring.
* Add "extension.proto" to defined the contract between client (Bicep)
and extension in gRPC.
* Support unpacking of provider packages to the local file system to the
~/.bicep cache folder.

The full spec is available
[here](https://microsoft.sharepoint.com/:w:/t/UniZomb/EVjskctj231Ipua9EVu5e4oBV83O8WsxNpD3InQZevLe2g?e=1CoJeV)
- unfortunately only available to Microsoft employees currently.

###### Microsoft Reviewers: [Open in
CodeFlow](https://microsoft.github.io/open-pr/?codeflow=https://github.com/Azure/bicep/pull/14231)

Bicep.sln

@@ -83,6 +83,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tools", "Tools", "{C6CAFEE8
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Bicep.Tools.Benchmark", "src\Bicep.Tools.Benchmark\Bicep.Tools.Benchmark.csproj", "{A4127F6F-A282-47F3-BAFE-6A154F994B4E}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{8F8DCFBC-A0DC-4E40-93C8-B4FB99FBD757}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Bicep.Local.Extension", "src\Bicep.Local.Extension\Bicep.Local.Extension.csproj", "{E84C0368-0D02-4284-A3CB-110B14FA8314}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -169,6 +173,10 @@ Global
 		{A4127F6F-A282-47F3-BAFE-6A154F994B4E}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{A4127F6F-A282-47F3-BAFE-6A154F994B4E}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{A4127F6F-A282-47F3-BAFE-6A154F994B4E}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E84C0368-0D02-4284-A3CB-110B14FA8314}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E84C0368-0D02-4284-A3CB-110B14FA8314}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E84C0368-0D02-4284-A3CB-110B14FA8314}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E84C0368-0D02-4284-A3CB-110B14FA8314}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -194,6 +202,7 @@ Global
 		{8C2280F2-1E08-4022-B26B-59622DD8B126} = {90126655-7A34-4F5B-A41A-50A468F4632D}
 		{9F596D8D-5CDB-4830-B73C-26C33C0227D7} = {90126655-7A34-4F5B-A41A-50A468F4632D}
 		{A4127F6F-A282-47F3-BAFE-6A154F994B4E} = {C6CAFEE8-7779-4F48-BEA1-D59D3CD91A56}
+		{E84C0368-0D02-4284-A3CB-110B14FA8314} = {8F8DCFBC-A0DC-4E40-93C8-B4FB99FBD757}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {21F77282-91E7-4304-B1EF-FADFA4F39E37}

docs/experimental-features.md

@@ -21,6 +21,9 @@ Allows Bicep to use a provider model to deploy non-ARM resources. Currently, we
 ### `legacyFormatter`
 Enables code formatting with the legacy formatter. This feature flag is introduced to ensure a safer transition to the v2 formatter that implements a pretty-printing algorithm. It is intended for temporary use and will be phased out soon.
 
+### `localDeploy`
+Enables local deployment capability. See [Bicep Local Providers](https://github.com/anthony-c-martin/bicep-local-providers) for more information.
+
 ### `optionalModuleNames`
 Enabling this feature makes the `name` property in the body of `module` declarations optional. When a `module` omits the `name` property with the feature enabled, the Bicep compiler will automatically generate an expression for the name of the resulting nested deployment in the JSON. If you specify the `name` property, the compiler will use the specified expression in the resulting JSON. For more information, see [Added optional module names as an experimental feature](https://github.com/Azure/bicep/pull/12600).
 

src/Bicep.Cli/Commands/PublishProviderCommand.cs

@@ -1,10 +1,12 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Collections.Immutable;
 using System.IO.Abstractions;
 using Bicep.Cli.Arguments;
 using Bicep.Core.Diagnostics;
 using Bicep.Core.Exceptions;
+using Bicep.Core.Extensions;
 using Bicep.Core.Features;
 using Bicep.Core.FileSystem;
 using Bicep.Core.Modules;
@@ -57,11 +59,13 @@ public async Task<int> RunAsync(PublishProviderArguments args)
                 throw new BicepException($"Provider package creation failed: {exception.Message}");
             }
 
-            await this.PublishProviderAsync(providerReference, tarPayload, overwriteIfExists);
+            var package = new ProviderPackage(Types: tarPayload);
+
+            await this.PublishProviderAsync(providerReference, package, overwriteIfExists);
             return 0;
         }
 
-        private async Task PublishProviderAsync(ArtifactReference target, BinaryData tarPayload, bool overwriteIfExists)
+        private async Task PublishProviderAsync(ArtifactReference target, ProviderPackage package, bool overwriteIfExists)
         {
             try
             {
@@ -70,7 +74,7 @@ private async Task PublishProviderAsync(ArtifactReference target, BinaryData tar
                 {
                     throw new BicepException($"The Provider \"{target.FullyQualifiedReference}\" already exists. Use --force to overwrite the existing provider.");
                 }
-                await this.moduleDispatcher.PublishProvider(target, tarPayload);
+                await this.moduleDispatcher.PublishProvider(target, package);
             }
             catch (ExternalArtifactException exception)
             {

src/Bicep.Core.IntegrationTests/RegistryProviderTests.cs

@@ -106,7 +106,9 @@ public async Task Providers_published_to_a_registry_can_be_compiled()
     [TestMethod]
     public async Task Providers_published_to_filesystem_can_be_compiled()
     {
-        var services = new ServiceBuilder().WithFeatureOverrides(new(ExtensibilityEnabled: true, ProviderRegistry: true));
+        var cacheDirectory = FileHelper.GetCacheRootPath(TestContext);
+        Directory.CreateDirectory(cacheDirectory);
+        var services = new ServiceBuilder().WithFeatureOverrides(new(CacheRootDirectory: cacheDirectory, ExtensibilityEnabled: true, ProviderRegistry: true));
 
         var typesTgz = ThirdPartyTypeHelper.GetTestTypesTgz();
         var tempDirectory = FileHelper.GetUniqueTestOutputPath(TestContext);
@@ -129,7 +131,11 @@ await File.WriteAllTextAsync(bicepPath, """
 
         var bicepUri = PathHelper.FilePathToFileUrl(bicepPath);
 
-        var result = CompilationHelper.Compile(services, BicepTestConstants.FileResolver, [bicepUri], bicepUri);
+
+        var compiler = services.Build().GetCompiler();
+        var compilation = await compiler.CreateCompilation(bicepUri);
+
+        var result = CompilationHelper.GetCompilationResult(compilation);
 
         result.Should().NotHaveAnyDiagnostics();
     }

src/Bicep.Core.UnitTests/Configuration/ConfigurationManagerTests.cs

@@ -116,6 +116,7 @@ public void GetBuiltInConfiguration_NoParameter_ReturnsBuiltInConfigurationWithA
           "dynamicTypeLoading": false,
           "providerRegistry": false,
           "optionalModuleNames": false,
+          "localDeploy": false,
           "resourceDerivedTypes": false
         },
         "formatting": {
@@ -204,6 +205,7 @@ public void GetBuiltInConfiguration_DisableAllAnalyzers_ReturnsBuiltInConfigurat
           "dynamicTypeLoading": false,
           "providerRegistry": false,
           "optionalModuleNames": false,
+          "localDeploy": false,
           "resourceDerivedTypes": false
         },
         "formatting": {
@@ -317,6 +319,7 @@ public void GetBuiltInConfiguration_DisableAnalyzers_ReturnsBuiltInConfiguration
           "dynamicTypeLoading": false,
           "providerRegistry": false,
           "optionalModuleNames": false,
+          "localDeploy": false,
           "resourceDerivedTypes": false
         },
         "formatting": {
@@ -758,6 +761,7 @@ public void GetConfiguration_ValidCustomConfiguration_OverridesBuiltInConfigurat
           "dynamicTypeLoading": false,
           "providerRegistry": false,
           "optionalModuleNames": false,
+          "localDeploy": false,
           "resourceDerivedTypes": false
         },
         "formatting": {

src/Bicep.Core.UnitTests/Features/FeatureProviderOverrides.cs

@@ -20,6 +20,7 @@ public record FeatureProviderOverrides(
     bool? DynamicTypeLoadingEnabled = default,
     bool? ProviderRegistry = default,
     bool? OptionalModuleNamesEnabled = default,
+    bool? LocalDeployEnabled = default,
     bool? ResourceDerivedTypesEnabled = default,
     string? AssemblyVersion = BicepTestConstants.DevAssemblyFileVersion)
 {
@@ -37,6 +38,7 @@ public FeatureProviderOverrides(
         bool? DynamicTypeLoadingEnabled = default,
         bool? ProviderRegistry = default,
         bool? OptionalModuleNamesEnabled = default,
+        bool? LocalDeployEnabled = default,
         bool? ResourceDerivedTypesEnabled = default,
         string? AssemblyVersion = BicepTestConstants.DevAssemblyFileVersion
     ) : this(
@@ -53,6 +55,7 @@ public FeatureProviderOverrides(
         DynamicTypeLoadingEnabled,
         ProviderRegistry,
         OptionalModuleNamesEnabled,
+        LocalDeployEnabled,
         ResourceDerivedTypesEnabled,
         AssemblyVersion)
     { }

src/Bicep.Core.UnitTests/Features/OverriddenFeatureProvider.cs

@@ -40,5 +40,7 @@ public OverriddenFeatureProvider(IFeatureProvider features, FeatureProviderOverr
 
     public bool OptionalModuleNamesEnabled => overrides.OptionalModuleNamesEnabled ?? features.OptionalModuleNamesEnabled;
 
+    public bool LocalDeployEnabled => overrides.LocalDeployEnabled ?? features.LocalDeployEnabled;
+
     public bool ResourceDerivedTypesEnabled => overrides.ResourceDerivedTypesEnabled ?? features.ResourceDerivedTypesEnabled;
 }

src/Bicep.Core.UnitTests/Modules/LocalModuleReferenceTests.cs

@@ -3,6 +3,7 @@
 
 using Bicep.Core.FileSystem;
 using Bicep.Core.Modules;
+using Bicep.Core.Registry;
 using FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -47,7 +48,7 @@ public void TryParseModuleReference_ValidLocalReference_ShouldParse(string value
 
         private static LocalModuleReference Parse(string package)
         {
-            LocalModuleReference.TryParse(package, PathHelper.FilePathToFileUrl(Path.GetTempFileName())).IsSuccess(out var parsed, out var failureBuilder);
+            LocalModuleReference.TryParse(ArtifactType.Module, package, PathHelper.FilePathToFileUrl(Path.GetTempFileName())).IsSuccess(out var parsed, out var failureBuilder);
             parsed.Should().NotBeNull();
             failureBuilder.Should().BeNull();
             return parsed!;

src/Bicep.Core.UnitTests/Utils/CompilationHelper.cs

@@ -62,6 +62,11 @@ public static async Task<CompilationResult> RestoreAndCompile(ServiceBuilder ser
 
             var (uriDictionary, entryUri) = CreateFileDictionary(filesToAppend, "main.bicep");
 
+            return await RestoreAndCompile(services, uriDictionary, entryUri);
+        }
+
+        public static async Task<CompilationResult> RestoreAndCompile(ServiceBuilder services, IReadOnlyDictionary<Uri, string> uriDictionary, Uri entryUri)
+        {
             var compiler = services.Build().GetCompiler();
             var compilation = await compiler.CreateCompilation(entryUri, CreateWorkspace(uriDictionary));
 

src/Bicep.Core.UnitTests/Utils/RegistryHelper.cs

@@ -153,7 +153,7 @@ public static async Task PublishProviderToRegistryAsync(IDependencyHelper servic
             throw new InvalidOperationException($"Failed to get reference '{errorBuilder(DiagnosticBuilder.ForDocumentStart()).Message}'.");
         }
 
-        await dispatcher.PublishProvider(targetReference, tgzData);
+        await dispatcher.PublishProvider(targetReference, new(tgzData));
     }
 
     private static Uri RandomFileUri() => PathHelper.FilePathToFileUrl(Path.GetTempFileName());

src/Bicep.Core/Configuration/ExperimentalFeaturesEnabled.cs

@@ -18,6 +18,7 @@ public record ExperimentalFeaturesEnabled(
     bool DynamicTypeLoading,
     bool ProviderRegistry,
     bool OptionalModuleNames,
+    bool LocalDeploy,
     bool ResourceDerivedTypes)
 {
     public static ExperimentalFeaturesEnabled Bind(JsonElement element)

src/Bicep.Core/Emit/TemplateWriter.cs

@@ -998,7 +998,7 @@ private void EmitProviders(ExpressionEmitter emitter, ImmutableArray<DeclaredPro
             {
                 foreach (var provider in providers)
                 {
-                    var settings = provider.NamespaceType.Settings;
+                    var settings = provider.Settings;
 
                     emitter.EmitObjectProperty(provider.Name, () =>
                     {

src/Bicep.Core/Features/FeatureProvider.cs

@@ -42,6 +42,8 @@ public FeatureProvider(RootConfiguration configuration)
 
         public bool OptionalModuleNamesEnabled => configuration.ExperimentalFeaturesEnabled.OptionalModuleNames;
 
+        public bool LocalDeployEnabled => configuration.ExperimentalFeaturesEnabled.LocalDeploy;
+
         public bool ResourceDerivedTypesEnabled => configuration.ExperimentalFeaturesEnabled.ResourceDerivedTypes;
 
         private static bool ReadBooleanEnvVar(string envVar, bool defaultValue)

src/Bicep.Core/Features/IFeatureProvider.cs

@@ -30,6 +30,8 @@ public interface IFeatureProvider
 
     bool OptionalModuleNamesEnabled { get; }
 
+    bool LocalDeployEnabled { get; }
+
     bool ResourceDerivedTypesEnabled { get; }
 
     IEnumerable<(string name, bool impactsCompilation, bool usesExperimentalArmEngineFeature)> EnabledFeatureMetadata
@@ -49,6 +51,7 @@ public interface IFeatureProvider
                 (TestFrameworkEnabled, CoreResources.ExperimentalFeatureNames_TestFramework, false, false),
                 (AssertsEnabled, CoreResources.ExperimentalFeatureNames_Asserts, true, true),
                 (OptionalModuleNamesEnabled, CoreResources.ExperimentalFeatureNames_OptionalModuleNames, true, false),
+                (LocalDeployEnabled, "Enable local deploy", false, false),
                 (ResourceDerivedTypesEnabled, CoreResources.ExperimentalFeatureNames_ResourceDerivedTypes, true, false),
             })
             {

src/Bicep.Core/FileSystem/PathHelper.cs

@@ -247,5 +247,8 @@ public static string GetRelativePath(Uri source, Uri target)
 
             return relativeUri;
         }
+
+        public static Uri ResolveFilePath(Uri parentFileUri, string childFilePath)
+            => TryResolveFilePath(parentFileUri, childFilePath) ?? throw new InvalidOperationException($"Failed to resolve file path for URI {parentFileUri} and child path {childFilePath}");
     }
 }

src/Bicep.Core/Intermediate/Expression.cs

@@ -391,7 +391,7 @@ public override void Accept(IExpressionVisitor visitor)
 public record DeclaredProviderExpression(
     SyntaxBase? SourceSyntax,
     string Name,
-    NamespaceType NamespaceType,
+    NamespaceSettings Settings,
     Expression? Config,
     Expression? Description = null
 ) : DescribableExpression(SourceSyntax, Description)

src/Bicep.Core/Intermediate/ExpressionBuilder.cs

@@ -174,7 +174,7 @@ private Expression ConvertWithoutLowering(SyntaxBase syntax)
                 return EvaluateDecorators(provider, new DeclaredProviderExpression(
                     provider,
                     symbol.Name,
-                    GetTypeInfo<NamespaceType>(provider),
+                    GetTypeInfo<NamespaceType>(provider).Settings,
                     provider.Config is not null ? ConvertWithoutLowering(provider.Config) : null));
 
             case ParameterDeclarationSyntax parameter:

src/Bicep.Core/Modules/LocalModuleReference.cs

@@ -15,12 +15,15 @@ public class LocalModuleReference : ArtifactReference
     {
         private static readonly IEqualityComparer<string> PathComparer = StringComparer.Ordinal;
 
-        private LocalModuleReference(string path, Uri parentModuleUri)
+        private LocalModuleReference(ArtifactType artifactType, string path, Uri parentModuleUri)
             : base(ArtifactReferenceSchemes.Local, parentModuleUri)
         {
+            ArtifactType = artifactType;
             this.Path = path;
         }
 
+        public ArtifactType ArtifactType { get; }
+
         /// <summary>
         /// Gets the relative path to the module.
         /// </summary>
@@ -44,10 +47,10 @@ public override bool Equals(object? obj)
 
         public override bool IsExternal => false;
 
-        public static ResultWithDiagnostic<LocalModuleReference> TryParse(string unqualifiedReference, Uri parentModuleUri)
+        public static ResultWithDiagnostic<LocalModuleReference> TryParse(ArtifactType artifactType, string unqualifiedReference, Uri parentModuleUri)
         {
             return Validate(unqualifiedReference)
-                .Transform(_ => new LocalModuleReference(unqualifiedReference, parentModuleUri));
+                .Transform(_ => new LocalModuleReference(artifactType, unqualifiedReference, parentModuleUri));
         }
 
         public static ResultWithDiagnostic<bool> Validate(string pathName)

src/Bicep.Core/Registry/ArtifactRegistry.cs

@@ -22,7 +22,7 @@ public RegistryCapabilities GetCapabilities(ArtifactType artifactType, ArtifactR
 
         public abstract Task PublishModule(T reference, BinaryData compiled, BinaryData? bicepSources, string? documentationUri, string? description);
 
-        public abstract Task PublishProvider(T reference, BinaryData typesTgz);
+        public abstract Task PublishProvider(T reference, ProviderPackage provider);
 
         public abstract Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<T> references);
 
@@ -46,8 +46,8 @@ public Task<bool> CheckArtifactExists(ArtifactType artifactType, ArtifactReferen
         public Task PublishModule(ArtifactReference artifactReference, BinaryData compiled, BinaryData? bicepSources, string? documentationUri, string? description)
             => this.PublishModule(ConvertReference(artifactReference), compiled, bicepSources, documentationUri, description);
 
-        public Task PublishProvider(ArtifactReference reference, BinaryData typesTgz)
-            => this.PublishProvider(ConvertReference(reference), typesTgz);
+        public Task PublishProvider(ArtifactReference reference, ProviderPackage provider)
+            => this.PublishProvider(ConvertReference(reference), provider);
 
         public Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<ArtifactReference> references) =>
             this.RestoreArtifacts(references.Select(ConvertReference));

src/Bicep.Core/Registry/DefaultArtifactRegistryProvider.cs

@@ -44,8 +44,7 @@ public ImmutableArray<IArtifactRegistry> Registries(Uri templateUri)
             var builder = ImmutableArray.CreateBuilder<IArtifactRegistry>();
 
             // Using IServiceProvider instead of constructor injection due to a dependency cycle
-            var compiler = this.serviceProvider.GetService<BicepCompiler>();
-            builder.Add(new LocalModuleRegistry(this.fileResolver, templateUri, compiler));
+            builder.Add(new LocalModuleRegistry(fileResolver, fileSystem, features, templateUri));
             builder.Add(new OciArtifactRegistry(this.fileResolver, this.fileSystem, this.clientFactory, features, configuration, templateUri));
             builder.Add(new TemplateSpecModuleRegistry(this.fileResolver, this.fileSystem, this.templateSpecRepositoryFactory, features, configuration, templateUri));
 

src/Bicep.Core/Registry/IArtifactDispatcher.cs

@@ -1,12 +1,16 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Collections.Immutable;
 using Bicep.Core.Diagnostics;
 using Bicep.Core.SourceCode;
 using Bicep.Core.Utils;
 
 namespace Bicep.Core.Registry
 {
+    public record ProviderPackage(
+        BinaryData Types);
+
     public interface IModuleDispatcher : IArtifactReferenceFactory
     {
         RegistryCapabilities GetRegistryCapabilities(ArtifactType artifactType, ArtifactReference reference);
@@ -23,7 +27,7 @@ public interface IModuleDispatcher : IArtifactReferenceFactory
 
         Task PublishModule(ArtifactReference reference, BinaryData compiledArmTemplate, BinaryData? bicepSources, string? documentationUri);
 
-        Task PublishProvider(ArtifactReference reference, BinaryData compiledArmTemplate);
+        Task PublishProvider(ArtifactReference reference, ProviderPackage provider);
 
         void PruneRestoreStatuses();
 

src/Bicep.Core/Registry/IArtifactRegistry.cs

@@ -77,8 +77,7 @@ public interface IArtifactRegistry
         /// Publishes a provider types package to the registry.
         /// </summary>
         /// <param name="reference">The provider reference</param>
-        /// <param name="typesTgz">A BinaryData object with the contents of the types.tgz</param>
-        Task PublishProvider(ArtifactReference reference, BinaryData typesTgz);
+        Task PublishProvider(ArtifactReference reference, ProviderPackage provider);
 
         /// <summary>
         /// Returns documentationUri for the module.