From b6f869f8813438016ae0ff1d5c29fbda08c65ca0 Mon Sep 17 00:00:00 2001
From: Aishwarya Bhandari <37918412+aishwaryabh@users.noreply.github.com>
Date: Wed, 2 Oct 2024 15:12:21 -0700
Subject: [PATCH] Add traits for e2e tests to use in final pipeline (#3841)

* adding trait for e2e tests to use in final pipeline

* test changes

* removing duplicate test

* fixing tests

* trying to filter tests again

* addressing comments

* adding extra label

* changing spacing back

* trying spacing again

* changing spacing back

* four spaces

* adding trait

* getting build to work

* removing constant

* addressing comments

* group trait
---
 .../Azure.Functions.Cli.Tests.csproj          |  4 +
 .../E2E/Helpers/TestTraits.cs                 | 29 ++++++
 .../E2E/StartTests.cs                         | 95 ++++++++++++++-----
 ...rtTests_artifact_consolidation.runsettings |  6 ++
 .../E2E/StartTests_default.runsettings        |  6 ++
 5 files changed, 115 insertions(+), 25 deletions(-)
 create mode 100644 test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs
 create mode 100644 test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings
 create mode 100644 test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings

diff --git a/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj b/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj
index f971ca791..9c8d9e31f 100644
--- a/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj
+++ b/test/Azure.Functions.Cli.Tests/Azure.Functions.Cli.Tests.csproj
@@ -48,4 +48,8 @@
   <ItemGroup>
     <ProjectReference Include="..\..\src\Azure.Functions.Cli\Azure.Functions.Cli.csproj" />
   </ItemGroup>
+
+  <PropertyGroup>
+    <RunSettingsFilePath>$(MSBuildProjectDirectory)\E2E\StartTests_default.runsettings</RunSettingsFilePath>
+  </PropertyGroup>
 </Project>
diff --git a/test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs b/test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs
new file mode 100644
index 000000000..51d5a7265
--- /dev/null
+++ b/test/Azure.Functions.Cli.Tests/E2E/Helpers/TestTraits.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Azure.Functions.Cli.Tests.E2E.Helpers
+{
+    internal static class TestTraits
+    {
+        /// <summary>
+        /// Defines a group of tests to be run together. Useful for test isolation.
+        /// </summary>
+        public const string Group = "Group";
+
+        // Groups
+        /// <summary>
+        /// Tests with RequiresNestedInProcArtifacts label will not be run in the default scenario and only in the artifact consolidation pipeline
+        /// Otherwise tests with this label will fail in the PR/ official core tools pipelines since the nested inproc artifacts are not present.
+        /// </summary>
+        public const string RequiresNestedInProcArtifacts = "RequiresNestedInProcArtifacts";
+
+        /// <summary>
+        /// Tests with UseInConsolidatedArtifactGeneration label will be used in the default scenario and in the artifact consolidation pipeline
+        /// We still want to run these tests in the PR/ official core tools pipelines and in the artifact consolidation pipeline for a sanity check before publishing the artifacts.
+        /// </summary>
+        public const string UseInConsolidatedArtifactGeneration = "UseInConsolidatedArtifactGeneration";
+    }
+}
diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs
index e252f2cdc..431d6ae43 100644
--- a/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs
+++ b/test/Azure.Functions.Cli.Tests/E2E/StartTests.cs
@@ -575,6 +575,40 @@ await CliTester.Run(new RunConfiguration
             }, _output);
         }
 
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.RequiresNestedInProcArtifacts)]
+        public async Task start_dotnet6_inproc_without_specifying_runtime_e2e()
+        {
+            await CliTester.Run(new RunConfiguration
+            {
+                Commands = new[]
+                {
+                    "init . --worker-runtime dotnet --target-framework net6.0",
+                    "new --template Httptrigger --name HttpTrigger",
+                    "start --port 7073 --verbose"
+                },
+                ExpectExit = false,
+                Test = async (workingDir, p) =>
+                {
+                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
+                    {
+                        (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                        var response = await client.GetAsync("/api/HttpTrigger?name=Test");
+                        var result = await response.Content.ReadAsStringAsync();
+                        p.Kill();
+                        await Task.Delay(TimeSpan.FromSeconds(2));
+                        result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+                        if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                        {
+                            testOutputHelper.Output.Should().Contain("Starting child process for inproc6 model host.");
+                            testOutputHelper.Output.Should().Contain("Selected inproc6 host.");
+                        }
+                    }
+                },
+                CommandTimeout = TimeSpan.FromSeconds(900),
+            }, _output);
+        }
+
         [Fact]
         public async Task start_dotnet6_inproc_without_specifying_runtime()
         {
@@ -599,6 +633,40 @@ await CliTester.Run(new RunConfiguration
             }, _output);
         }
 
+        [Fact]
+        [Trait(TestTraits.Group, TestTraits.RequiresNestedInProcArtifacts)]
+        public async Task start_dotnet6_inproc_with_specifying_runtime_e2e()
+        {
+            await CliTester.Run(new RunConfiguration
+            {
+                Commands = new[]
+                {
+                    "init . --worker-runtime dotnet --target-framework net6.0",
+                    "new --template Httptrigger --name HttpTrigger",
+                    "start --port 7073 --verbose --runtime inproc6"
+                },
+                ExpectExit = false,
+                Test = async (workingDir, p) =>
+                {
+                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
+                    {
+                        (await WaitUntilReady(client)).Should().BeTrue(because: _serverNotReady);
+                        var response = await client.GetAsync("/api/HttpTrigger?name=Test");
+                        var result = await response.Content.ReadAsStringAsync();
+                        p.Kill();
+                        await Task.Delay(TimeSpan.FromSeconds(2));
+                        result.Should().Be("Hello, Test. This HTTP triggered function executed successfully.", because: "response from default function should be 'Hello, {name}. This HTTP triggered function executed successfully.'");
+                        if (_output is Xunit.Sdk.TestOutputHelper testOutputHelper)
+                        {
+                            testOutputHelper.Output.Should().Contain("Starting child process for inproc8 model host.");
+                            testOutputHelper.Output.Should().Contain("Selected inproc8 host.");
+                        }
+                    }
+                },
+                CommandTimeout = TimeSpan.FromSeconds(900),
+            }, _output);
+        }
+
         [Fact]
         public async Task start_dotnet6_inproc_with_specifying_runtime()
         {
@@ -816,31 +884,7 @@ await CliTester.Run(new RunConfiguration
         }
 
         [Fact]
-        public async Task start_dotnet_isolated_inproc_with_specifying_runtime()
-        {
-            await CliTester.Run(new RunConfiguration
-            {
-                Commands = new[]
-                {
-                    "init . --worker-runtime dotnet --target-framework net6.0",
-                    "new --template Httptrigger --name HttpTrigger",
-                    "start --port 7073 --verbose --runtime inproc6"
-                },
-                ExpectExit = false,
-                ErrorContains = ["Failed to locate the inproc6 model host at"],
-                Test = async (workingDir, p) =>
-                {
-                    using (var client = new HttpClient() { BaseAddress = new Uri("http://localhost:7073") })
-                    {
-                        await Task.Delay(TimeSpan.FromSeconds(2));
-                    }
-                },
-                CommandTimeout = TimeSpan.FromSeconds(100),
-            }, _output);
-        }
-
-
-        [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInConsolidatedArtifactGeneration)]
         public async Task start_dotnet_isolated_csharp_with_oop_host_with_runtime_specified()
         {
             await CliTester.Run(new RunConfiguration
@@ -875,6 +919,7 @@ await CliTester.Run(new RunConfiguration
         }
 
         [Fact]
+        [Trait(TestTraits.Group, TestTraits.UseInConsolidatedArtifactGeneration)]
         public async Task start_dotnet_isolated_csharp_with_oop_host_without_runtime_specified()
         {
             await CliTester.Run(new RunConfiguration
diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings b/test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings
new file mode 100644
index 000000000..256cfd64e
--- /dev/null
+++ b/test/Azure.Functions.Cli.Tests/E2E/StartTests_artifact_consolidation.runsettings
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <TestCaseFilter>(Group = RequiresNestedInProcArtifacts | Group = UseInConsolidatedArtifactGeneration)</TestCaseFilter>
+  </RunConfiguration>
+</RunSettings>
\ No newline at end of file
diff --git a/test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings b/test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings
new file mode 100644
index 000000000..8a7f214e9
--- /dev/null
+++ b/test/Azure.Functions.Cli.Tests/E2E/StartTests_default.runsettings
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <TestCaseFilter>(Group != RequiresNestedInProcArtifacts)</TestCaseFilter>
+  </RunConfiguration>
+</RunSettings>
\ No newline at end of file