More wiring to support providers package containing more than just types

src/Bicep.Cli/Commands/PublishProviderCommand.cs

@@ -1,10 +1,12 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Collections.Immutable;
 using System.IO.Abstractions;
 using Bicep.Cli.Arguments;
 using Bicep.Core.Diagnostics;
 using Bicep.Core.Exceptions;
+using Bicep.Core.Extensions;
 using Bicep.Core.Features;
 using Bicep.Core.FileSystem;
 using Bicep.Core.Modules;
@@ -57,11 +59,13 @@ public async Task<int> RunAsync(PublishProviderArguments args)
                 throw new BicepException($"Provider package creation failed: {exception.Message}");
             }
 
-            await this.PublishProviderAsync(providerReference, tarPayload, overwriteIfExists);
+            var package = new ProviderPackage(Types: tarPayload);
+
+            await this.PublishProviderAsync(providerReference, package, overwriteIfExists);
             return 0;
         }
 
-        private async Task PublishProviderAsync(ArtifactReference target, BinaryData tarPayload, bool overwriteIfExists)
+        private async Task PublishProviderAsync(ArtifactReference target, ProviderPackage package, bool overwriteIfExists)
         {
             try
             {
@@ -70,7 +74,7 @@ private async Task PublishProviderAsync(ArtifactReference target, BinaryData tar
                 {
                     throw new BicepException($"The Provider \"{target.FullyQualifiedReference}\" already exists. Use --force to overwrite the existing provider.");
                 }
-                await this.moduleDispatcher.PublishProvider(target, tarPayload);
+                await this.moduleDispatcher.PublishProvider(target, package);
             }
             catch (ExternalArtifactException exception)
             {

src/Bicep.Core.IntegrationTests/RegistryProviderTests.cs

@@ -106,7 +106,9 @@ public async Task Providers_published_to_a_registry_can_be_compiled()
     [TestMethod]
     public async Task Providers_published_to_filesystem_can_be_compiled()
     {
-        var services = new ServiceBuilder().WithFeatureOverrides(new(ExtensibilityEnabled: true, ProviderRegistry: true));
+        var cacheDirectory = FileHelper.GetCacheRootPath(TestContext);
+        Directory.CreateDirectory(cacheDirectory);
+        var services = new ServiceBuilder().WithFeatureOverrides(new(CacheRootDirectory: cacheDirectory, ExtensibilityEnabled: true, ProviderRegistry: true));
 
         var typesTgz = ThirdPartyTypeHelper.GetTestTypesTgz();
         var tempDirectory = FileHelper.GetUniqueTestOutputPath(TestContext);
@@ -129,7 +131,11 @@ await File.WriteAllTextAsync(bicepPath, """
 
         var bicepUri = PathHelper.FilePathToFileUrl(bicepPath);
 
-        var result = CompilationHelper.Compile(services, BicepTestConstants.FileResolver, [bicepUri], bicepUri);
+
+        var compiler = services.Build().GetCompiler();
+        var compilation = await compiler.CreateCompilation(bicepUri);
+
+        var result = CompilationHelper.GetCompilationResult(compilation);
 
         result.Should().NotHaveAnyDiagnostics();
     }

src/Bicep.Core.UnitTests/Modules/LocalModuleReferenceTests.cs

@@ -3,6 +3,7 @@
 
 using Bicep.Core.FileSystem;
 using Bicep.Core.Modules;
+using Bicep.Core.Registry;
 using FluentAssertions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
@@ -47,7 +48,7 @@ public void TryParseModuleReference_ValidLocalReference_ShouldParse(string value
 
         private static LocalModuleReference Parse(string package)
         {
-            LocalModuleReference.TryParse(package, PathHelper.FilePathToFileUrl(Path.GetTempFileName())).IsSuccess(out var parsed, out var failureBuilder);
+            LocalModuleReference.TryParse(ArtifactType.Module, package, PathHelper.FilePathToFileUrl(Path.GetTempFileName())).IsSuccess(out var parsed, out var failureBuilder);
             parsed.Should().NotBeNull();
             failureBuilder.Should().BeNull();
             return parsed!;

src/Bicep.Core.UnitTests/Utils/RegistryHelper.cs

@@ -153,7 +153,7 @@ public static async Task PublishProviderToRegistryAsync(IDependencyHelper servic
             throw new InvalidOperationException($"Failed to get reference '{errorBuilder(DiagnosticBuilder.ForDocumentStart()).Message}'.");
         }
 
-        await dispatcher.PublishProvider(targetReference, tgzData);
+        await dispatcher.PublishProvider(targetReference, new(tgzData));
     }
 
     private static Uri RandomFileUri() => PathHelper.FilePathToFileUrl(Path.GetTempFileName());

src/Bicep.Core/Emit/TemplateWriter.cs

@@ -998,7 +998,7 @@ private void EmitProviders(ExpressionEmitter emitter, ImmutableArray<DeclaredPro
             {
                 foreach (var provider in providers)
                 {
-                    var settings = provider.NamespaceType.Settings;
+                    var settings = provider.Settings;
 
                     emitter.EmitObjectProperty(provider.Name, () =>
                     {

src/Bicep.Core/Intermediate/Expression.cs

@@ -391,7 +391,7 @@ public override void Accept(IExpressionVisitor visitor)
 public record DeclaredProviderExpression(
     SyntaxBase? SourceSyntax,
     string Name,
-    NamespaceType NamespaceType,
+    NamespaceSettings Settings,
     Expression? Config,
     Expression? Description = null
 ) : DescribableExpression(SourceSyntax, Description)

src/Bicep.Core/Intermediate/ExpressionBuilder.cs

@@ -174,7 +174,7 @@ private Expression ConvertWithoutLowering(SyntaxBase syntax)
                 return EvaluateDecorators(provider, new DeclaredProviderExpression(
                     provider,
                     symbol.Name,
-                    GetTypeInfo<NamespaceType>(provider),
+                    GetTypeInfo<NamespaceType>(provider).Settings,
                     provider.Config is not null ? ConvertWithoutLowering(provider.Config) : null));
 
             case ParameterDeclarationSyntax parameter:

src/Bicep.Core/Modules/LocalModuleReference.cs

@@ -15,12 +15,15 @@ public class LocalModuleReference : ArtifactReference
     {
         private static readonly IEqualityComparer<string> PathComparer = StringComparer.Ordinal;
 
-        private LocalModuleReference(string path, Uri parentModuleUri)
+        private LocalModuleReference(ArtifactType artifactType, string path, Uri parentModuleUri)
             : base(ArtifactReferenceSchemes.Local, parentModuleUri)
         {
+            ArtifactType = artifactType;
             this.Path = path;
         }
 
+        public ArtifactType ArtifactType { get; }
+
         /// <summary>
         /// Gets the relative path to the module.
         /// </summary>
@@ -44,10 +47,10 @@ public override bool Equals(object? obj)
 
         public override bool IsExternal => false;
 
-        public static ResultWithDiagnostic<LocalModuleReference> TryParse(string unqualifiedReference, Uri parentModuleUri)
+        public static ResultWithDiagnostic<LocalModuleReference> TryParse(ArtifactType artifactType, string unqualifiedReference, Uri parentModuleUri)
         {
             return Validate(unqualifiedReference)
-                .Transform(_ => new LocalModuleReference(unqualifiedReference, parentModuleUri));
+                .Transform(_ => new LocalModuleReference(artifactType, unqualifiedReference, parentModuleUri));
         }
 
         public static ResultWithDiagnostic<bool> Validate(string pathName)

src/Bicep.Core/Registry/ArtifactRegistry.cs

@@ -22,7 +22,7 @@ public RegistryCapabilities GetCapabilities(ArtifactType artifactType, ArtifactR
 
         public abstract Task PublishModule(T reference, BinaryData compiled, BinaryData? bicepSources, string? documentationUri, string? description);
 
-        public abstract Task PublishProvider(T reference, BinaryData typesTgz);
+        public abstract Task PublishProvider(T reference, ProviderPackage provider);
 
         public abstract Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<T> references);
 
@@ -46,8 +46,8 @@ public Task<bool> CheckArtifactExists(ArtifactType artifactType, ArtifactReferen
         public Task PublishModule(ArtifactReference artifactReference, BinaryData compiled, BinaryData? bicepSources, string? documentationUri, string? description)
             => this.PublishModule(ConvertReference(artifactReference), compiled, bicepSources, documentationUri, description);
 
-        public Task PublishProvider(ArtifactReference reference, BinaryData typesTgz)
-            => this.PublishProvider(ConvertReference(reference), typesTgz);
+        public Task PublishProvider(ArtifactReference reference, ProviderPackage provider)
+            => this.PublishProvider(ConvertReference(reference), provider);
 
         public Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<ArtifactReference> references) =>
             this.RestoreArtifacts(references.Select(ConvertReference));

src/Bicep.Core/Registry/DefaultArtifactRegistryProvider.cs

@@ -44,8 +44,7 @@ public ImmutableArray<IArtifactRegistry> Registries(Uri templateUri)
             var builder = ImmutableArray.CreateBuilder<IArtifactRegistry>();
 
             // Using IServiceProvider instead of constructor injection due to a dependency cycle
-            var compiler = this.serviceProvider.GetService<BicepCompiler>();
-            builder.Add(new LocalModuleRegistry(this.fileResolver, templateUri, compiler));
+            builder.Add(new LocalModuleRegistry(fileResolver, fileSystem, features, templateUri));
             builder.Add(new OciArtifactRegistry(this.fileResolver, this.fileSystem, this.clientFactory, features, configuration, templateUri));
             builder.Add(new TemplateSpecModuleRegistry(this.fileResolver, this.fileSystem, this.templateSpecRepositoryFactory, features, configuration, templateUri));
 

src/Bicep.Core/Registry/IArtifactDispatcher.cs

@@ -1,12 +1,16 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System.Collections.Immutable;
 using Bicep.Core.Diagnostics;
 using Bicep.Core.SourceCode;
 using Bicep.Core.Utils;
 
 namespace Bicep.Core.Registry
 {
+    public record ProviderPackage(
+        BinaryData Types);
+
     public interface IModuleDispatcher : IArtifactReferenceFactory
     {
         RegistryCapabilities GetRegistryCapabilities(ArtifactType artifactType, ArtifactReference reference);
@@ -23,7 +27,7 @@ public interface IModuleDispatcher : IArtifactReferenceFactory
 
         Task PublishModule(ArtifactReference reference, BinaryData compiledArmTemplate, BinaryData? bicepSources, string? documentationUri);
 
-        Task PublishProvider(ArtifactReference reference, BinaryData compiledArmTemplate);
+        Task PublishProvider(ArtifactReference reference, ProviderPackage provider);
 
         void PruneRestoreStatuses();
 

src/Bicep.Core/Registry/IArtifactRegistry.cs

@@ -77,8 +77,7 @@ public interface IArtifactRegistry
         /// Publishes a provider types package to the registry.
         /// </summary>
         /// <param name="reference">The provider reference</param>
-        /// <param name="typesTgz">A BinaryData object with the contents of the types.tgz</param>
-        Task PublishProvider(ArtifactReference reference, BinaryData typesTgz);
+        Task PublishProvider(ArtifactReference reference, ProviderPackage provider);
 
         /// <summary>
         /// Returns documentationUri for the module.

src/Bicep.Core/Registry/LocalModuleRegistry.cs

@@ -1,32 +1,33 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using System.Collections.Immutable;
 using System.Diagnostics;
-using System.IO;
-using System.Threading.Tasks;
 using Bicep.Core.Diagnostics;
-using Bicep.Core.Extensions;
 using Bicep.Core.FileSystem;
 using Bicep.Core.Modules;
 using Bicep.Core.Registry.Providers;
 using Bicep.Core.Semantics;
 using Bicep.Core.SourceCode;
 using Bicep.Core.Utils;
+using System.IO.Abstractions;
+using Bicep.Core.Features;
+using Bicep.Core.Registry.Oci;
+using System.Runtime.InteropServices;
 
 namespace Bicep.Core.Registry
 {
-    public class LocalModuleRegistry : ArtifactRegistry<LocalModuleReference>
+    public record LocalModuleEntity(ProviderPackage Provider);
+
+    public class LocalModuleRegistry : ExternalArtifactRegistry<LocalModuleReference, LocalModuleEntity>
     {
-        private readonly IFileResolver fileResolver;
+        private readonly IFeatureProvider featureProvider;
         private readonly Uri parentModuleUri;
-        private readonly BicepCompiler? bicepCompiler;
 
-        public LocalModuleRegistry(IFileResolver fileResolver, Uri parentModuleUri, BicepCompiler? bicepCompiler)
+        public LocalModuleRegistry(IFileResolver fileResolver, IFileSystem fileSystem, IFeatureProvider featureProvider, Uri parentModuleUri)
+            : base(fileResolver, fileSystem)
         {
-            this.fileResolver = fileResolver;
+            this.featureProvider = featureProvider;
             this.parentModuleUri = parentModuleUri;
-            this.bicepCompiler = bicepCompiler;
         }
 
         public override string Scheme => ArtifactReferenceSchemes.Local;
@@ -46,7 +47,7 @@ public override ResultWithDiagnostic<ArtifactReference> TryParseArtifactReferenc
                 return new(x => x.UnsupportedArtifactType(artifactType));
             }
 
-            if (!LocalModuleReference.TryParse(reference, parentModuleUri).IsSuccess(out var @ref, out var failureBuilder))
+            if (!LocalModuleReference.TryParse(artifactType, reference, parentModuleUri).IsSuccess(out var @ref, out var failureBuilder))
             {
                 return new(failureBuilder);
             }
@@ -57,7 +58,12 @@ public override ResultWithDiagnostic<ArtifactReference> TryParseArtifactReferenc
 
         public override ResultWithDiagnostic<Uri> TryGetLocalArtifactEntryPointUri(LocalModuleReference reference)
         {
-            var localUri = fileResolver.TryResolveFilePath(reference.ParentModuleUri, reference.Path);
+            if (reference.ArtifactType == ArtifactType.Provider)
+            {
+                return new(GetTypesTgzUri(reference));
+            }
+
+            var localUri = FileResolver.TryResolveFilePath(reference.ParentModuleUri, reference.Path);
             if (localUri is null)
             {
                 return new(x => x.FilePathCouldNotBeResolved(reference.Path, reference.ParentModuleUri.LocalPath));
@@ -66,31 +72,52 @@ public override ResultWithDiagnostic<Uri> TryGetLocalArtifactEntryPointUri(Local
             return new(localUri);
         }
 
-        public override Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<LocalModuleReference> references)
+        public override async Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> RestoreArtifacts(IEnumerable<LocalModuleReference> references)
         {
-            // local modules are already present on the file system
-            // and do not require init
-            return Task.FromResult<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>>(ImmutableDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>.Empty);
+            var statuses = new Dictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>();
+
+            foreach (var reference in references)
+            {
+                if (reference.ArtifactType == ArtifactType.Provider)
+                {
+                    if (TryReadContent(reference) is not {} binaryData)
+                    {
+                        statuses.Add(reference, x => x.ArtifactRestoreFailedWithMessage(reference.FullyQualifiedReference, $"Failed to find {reference.FullyQualifiedReference}"));
+                        continue;
+                    }
+
+                    var package = ProviderV1Archive.Read(binaryData);
+                    await this.WriteArtifactContentToCacheAsync(reference, new(package));
+                }
+            }
+
+            return statuses;
         }
 
-        public override Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> InvalidateArtifactsCache(IEnumerable<LocalModuleReference> references)
+        public override async Task<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>> InvalidateArtifactsCache(IEnumerable<LocalModuleReference> references)
         {
-            // local modules are already present on the file system, there's no cache concept for this one
-            // we do nothing
-            return Task.FromResult<IDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>>(ImmutableDictionary<ArtifactReference, DiagnosticBuilder.ErrorBuilderDelegate>.Empty);
+            return await base.InvalidateArtifactsCacheInternal(references);
         }
 
-        public override bool IsArtifactRestoreRequired(LocalModuleReference reference) => false;
+        public override bool IsArtifactRestoreRequired(LocalModuleReference reference)
+        {
+            if (reference.ArtifactType != ArtifactType.Provider)
+            {
+                return false;
+            }
+
+            return !this.FileResolver.FileExists(this.GetTypesTgzUri(reference));
+        }
 
         public override Task PublishModule(LocalModuleReference moduleReference, BinaryData compiledArmTemplate, BinaryData? bicepSources, string? documentationUri, string? description)
             => throw new NotSupportedException("Local modules cannot be published.");
 
-        public override async Task PublishProvider(LocalModuleReference reference, BinaryData typesTgz)
+        public override async Task PublishProvider(LocalModuleReference reference, ProviderPackage provider)
         {
-            var archive = await ProviderV1Archive.Build(typesTgz);
+            var archive = await ProviderV1Archive.Build(provider);
 
             var fileUri = PathHelper.TryResolveFilePath(reference.ParentModuleUri, reference.Path)!;
-            fileResolver.Write(fileUri, archive.ToStream());
+            FileResolver.Write(fileUri, archive.ToStream());
         }
 
         public override Task<bool> CheckArtifactExists(ArtifactType artifactType, LocalModuleReference reference)
@@ -117,5 +144,46 @@ public override ResultWithException<SourceArchive> TryGetSource(LocalModuleRefer
         {
             return new(new SourceNotAvailableException());
         }
+
+        protected override void WriteArtifactContentToCache(LocalModuleReference reference, LocalModuleEntity entity)
+        {
+            var typesUri = this.GetTypesTgzUri(reference);
+            this.FileResolver.Write(typesUri, entity.Provider.Types.ToStream());
+        }
+
+        protected override string GetArtifactDirectoryPath(LocalModuleReference reference)
+        {
+            if (TryReadContent(reference) is not {} binaryData)
+            {
+                throw new InvalidOperationException($"Failed to resolve file path for {reference.FullyQualifiedReference}");
+            }
+
+            // Provider packages are unpacked to '~/.bicep/local/sha256_<digest>'.
+            // We must use '_' as a separator here because Windows does not allow ':' in file paths.
+            var digest = OciDescriptor.ComputeDigest(OciDescriptor.AlgorithmIdentifierSha256, binaryData, separator: '_');
+
+            return FileSystem.Path.Combine(
+                this.featureProvider.CacheRootDirectory,
+                "local",
+                digest);
+        }
+
+        private BinaryData? TryReadContent(LocalModuleReference reference)
+        {
+            if (FileResolver.TryResolveFilePath(reference.ParentModuleUri, reference.Path) is not {} fileUri ||
+                FileResolver.TryReadAsBinaryData(fileUri).TryUnwrap() is not {} binaryData)
+            {
+                return null;
+            }
+
+            return binaryData;
+        }
+
+        private Uri GetTypesTgzUri(LocalModuleReference reference) => GetFileUri(reference, "types.tgz");
+
+        protected override Uri GetArtifactLockFileUri(LocalModuleReference reference) => GetFileUri(reference, "lock");
+
+        private Uri GetFileUri(LocalModuleReference reference, string path)
+            => new(FileSystem.Path.Combine(this.GetArtifactDirectoryPath(reference), path), UriKind.Absolute);
     }
 }