Add traits for e2e tests to use in final pipeline (#3841)

* adding trait for e2e tests to use in final pipeline * test changes * removing duplicate test * fixing tests * trying to filter tests again * addressing comments * adding extra label * changing spacing back * trying spacing again * changing spacing back * four spaces * adding trait * getting build to work * removing constant * addressing comments * group trait
Azure · Nov 4, 2024 · b6f869f · b6f869f
1 parent 5b7f4c7
commit b6f869f
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 25 deletions.
diff --git a/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj b/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj
@@ -48,4 +48,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\Azure.Functions.Cli\Azure.Functions.Cli.csproj" />
   </ItemGroup>
+
+  <PropertyGroup>
+    <RunSettingsFilePath>$(MSBuildProjectDirectory)\E2E\StartTests_default.runsettings</RunSettingsFilePath>
+  </PropertyGroup>
 </Project>
diff --git a/test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs b/test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Azure.Functions.Cli.Tests.E2E.Helpers
+{
+    internal static class TestTraits
+    {
+        /// <summary>
+        /// Defines a group of tests to be run together. Useful for test isolation.
+        /// </summary>
+        public const string Group = "Group";
+
+        // Groups
+        /// <summary>
+        /// Tests with RequiresNestedInProcArtifacts label will not be run in the default scenario and only in the artifact consolidation pipeline
+        /// Otherwise tests with this label will fail in the PR/ official core tools pipelines since the nested inproc artifacts are not present.
+        /// </summary>
+        public const string RequiresNestedInProcArtifacts = "RequiresNestedInProcArtifacts";
+
+        /// <summary>
+        /// Tests with UseInConsolidatedArtifactGeneration label will be used in the default scenario and in the artifact consolidation pipeline
+        /// We still want to run these tests in the PR/ official core tools pipelines and in the artifact consolidation pipeline for a sanity check before publishing the artifacts.
+        /// </summary>
+        public const string UseInConsolidatedArtifactGeneration = "UseInConsolidatedArtifactGeneration";
+    }
+}
diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs
@@ -575,6 +575,40 @@ await CliTester.Run(new RunConfiguration
             }, _output);
         }
 
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.RequiresNestedInProcArtifacts)]
+        public async Task start_dotnet6_inproc_without_specifying_runtime_e2e()
+        {
+            await CliTester.Run(new RunConfiguration
+            {
+                Commands = new[]
+                {
+                    "init . --worker-runtime dotnet --target-framework net6.0",
+                    "new --template Httptrigger --name HttpTrigger",
+                    "start --port 7073 --verbose"
+                },
+                ExpectExit = false,
+                Test = async (workingDir, p) =>
+                {
+                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
+                    {
+                        (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                        var response = await client.GetAsync("/api/HttpTrigger?name=Test");
+                        var result = await response.Content.ReadAsStringAsync();
+                        p.Kill();
+                        await Task.Delay(TimeSpan.FromSeconds(2));
+                        result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+                        if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                        {
+                            testOutputHelper.Output.Should().Contain("Starting child process for inproc6 model host.");
+                            testOutputHelper.Output.Should().Contain("Selected inproc6 host.");
+                        }
+                    }
+                },
+                CommandTimeout = TimeSpan.FromSeconds(900),
+            }, _output);
+        }
+
         [Fact]
         public async Task start_dotnet6_inproc_without_specifying_runtime()
         {
@@ -599,6 +633,40 @@ await CliTester.Run(new RunConfiguration
             }, _output);
         }
 
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.RequiresNestedInProcArtifacts)]
+        public async Task start_dotnet6_inproc_with_specifying_runtime_e2e()
+        {
+            await CliTester.Run(new RunConfiguration
+            {
+                Commands = new[]
+                {
+                    "init . --worker-runtime dotnet --target-framework net6.0",
+                    "new --template Httptrigger --name HttpTrigger",
+                    "start --port 7073 --verbose --runtime inproc6"
+                },
+                ExpectExit = false,
+                Test = async (workingDir, p) =>
+                {
+                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
+                    {
+                        (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                        var response = await client.GetAsync("/api/HttpTrigger?name=Test");
+                        var result = await response.Content.ReadAsStringAsync();
+                        p.Kill();
+                        await Task.Delay(TimeSpan.FromSeconds(2));
+                        result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+                        if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                        {
+                            testOutputHelper.Output.Should().Contain("Starting child process for inproc8 model host.");
+                            testOutputHelper.Output.Should().Contain("Selected inproc8 host.");
+                        }
+                    }
+                },
+                CommandTimeout = TimeSpan.FromSeconds(900),
+            }, _output);
+        }
+
         [Fact]
         public async Task start_dotnet6_inproc_with_specifying_runtime()
         {
@@ -816,31 +884,7 @@ await CliTester.Run(new RunConfiguration
         }
 
         [Fact]
-        public async Task start_dotnet_isolated_inproc_with_specifying_runtime()
-        {
-            await CliTester.Run(new RunConfiguration
-            {
-                Commands = new[]
-                {
-                    "init . --worker-runtime dotnet --target-framework net6.0",
-                    "new --template Httptrigger --name HttpTrigger",
-                    "start --port 7073 --verbose --runtime inproc6"
-                },
-                ExpectExit = false,
-                ErrorContains = ["Failed to locate the inproc6 model host at"],
-                Test = async (workingDir, p) =>
-                {
-                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
-                    {
-                        await Task.Delay(TimeSpan.FromSeconds(2));
-                    }
-                },
-                CommandTimeout = TimeSpan.FromSeconds(100),
-            }, _output);
-        }
-
-
-        [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInConsolidatedArtifactGeneration)]
         public async Task start_dotnet_isolated_csharp_with_oop_host_with_runtime_specified()
         {
             await CliTester.Run(new RunConfiguration
@@ -875,6 +919,7 @@ await CliTester.Run(new RunConfiguration
         }
 
         [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInConsolidatedArtifactGeneration)]
         public async Task start_dotnet_isolated_csharp_with_oop_host_without_runtime_specified()
         {
             await CliTester.Run(new RunConfiguration

diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings b/test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <TestCaseFilter>(Group = RequiresNestedInProcArtifacts | Group = UseInConsolidatedArtifactGeneration)</TestCaseFilter>
+  </RunConfiguration>
+</RunSettings>
diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings b/test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <TestCaseFilter>(Group != RequiresNestedInProcArtifacts)</TestCaseFilter>
+  </RunConfiguration>
+</RunSettings>