Add MaxAttempts trait to retry-able tests

src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsFactDiscoverer.cs

@@ -38,7 +38,7 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
 
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
+            ? new UITestCase(UITestCase.SyncContextType.WinForms, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WinFormsTheoryDiscoverer.cs

@@ -23,15 +23,15 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFr
     {
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
+            ? new UITestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
     {
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
+            ? new UITheoryTestCase(UITestCase.SyncContextType.WinForms, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WinForms only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WpfFactDiscoverer.cs

@@ -38,7 +38,7 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
 
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
+            ? new UITestCase(UITestCase.SyncContextType.WPF, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk.WindowsDesktop/WpfTheoryDiscoverer.cs

@@ -23,15 +23,15 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFr
     {
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
+            ? new UITestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 
     protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFrameworkDiscoveryOptions discoveryOptions, ITestMethod testMethod, IAttributeInfo theoryAttribute)
     {
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
+            ? new UITheoryTestCase(UITestCase.SyncContextType.WPF, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "WPF only exists on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/StaFactDiscoverer.cs

@@ -39,7 +39,7 @@ protected override IXunitTestCase CreateTestCase(ITestFrameworkDiscoveryOptions
 
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
+            ? new UITestCase(UITestCase.SyncContextType.None, this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, testMethodArguments: null, settings)
             : new XunitSkippedDataRowTestCase(this.diagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/StaTheoryDiscoverer.cs

@@ -35,7 +35,7 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForDataRow(ITestFr
 
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
+            ? new UITestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), testMethod, dataRow, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 
@@ -53,7 +53,7 @@ protected override IEnumerable<IXunitTestCase> CreateTestCasesForTheory(ITestFra
 
         UISettingsAttribute settings = UIFactDiscoverer.GetSettings(testMethod);
         yield return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
-            ? (IXunitTestCase)new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
+            ? new UITheoryTestCase(UITestCase.SyncContextType.None, this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), TestMethodDisplayOptions.None, testMethod, settings)
             : new XunitSkippedDataRowTestCase(this.DiagnosticMessageSink, discoveryOptions.MethodDisplayOrDefault(), discoveryOptions.MethodDisplayOptionsOrDefault(), testMethod, "STA threads only exist on Windows.");
     }
 }

src/Xunit.StaFact/Sdk/UITestCase.cs

@@ -44,6 +44,7 @@ internal UITestCase(
         : base(diagnosticMessageSink, defaultMethodDisplay, TestMethodDisplayOptions.None, testMethod, testMethodArguments)
     {
         this.settings = settings;
+        settings.ApplyTraits(this);
         this.synchronizationContextType = synchronizationContextType;
     }
 

src/Xunit.StaFact/Sdk/UITheoryTestCase.cs

@@ -17,6 +17,7 @@ internal UITheoryTestCase(UITestCase.SyncContextType synchronizationContextType,
         : base(diagnosticMessageSink, defaultMethodDisplay, defaultMethodDisplayOptions, testMethod)
     {
         this.Settings = settings;
+        settings.ApplyTraits(this);
         this.SynchronizationContextType = synchronizationContextType;
     }
 

src/Xunit.StaFact/UISettingsAttribute.cs

@@ -1,6 +1,8 @@
 // Copyright (c) Andrew Arnott. All rights reserved.
 // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information.
 
+using Xunit.Sdk;
+
 namespace Xunit;
 
 [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = true)]
@@ -19,4 +21,19 @@ public sealed class UISettingsAttribute : Attribute
     public int MaxAttempts { get; set; }
 
     internal static UISettingsAttribute Default => new() { MaxAttempts = 1 };
+
+    /// <summary>
+    /// Applies traits to a test case based on the settings in this attribute.
+    /// </summary>
+    /// <param name="testCase">The test case to add traits to.</param>
+    internal void ApplyTraits(XunitTestCase testCase)
+    {
+        if (this.MaxAttempts > 1)
+        {
+            if (!testCase.Traits.ContainsKey("MaxAttempts"))
+            {
+                testCase.Traits.Add("MaxAttempts", new() { this.MaxAttempts.ToString() });
+            }
+        }
+    }
 }

test/Xunit.StaFact.Tests/MaxAttemptsHelper.cs

@@ -2,6 +2,7 @@
 // Licensed under the Ms-PL license. See LICENSE file in the project root for full license information.
 
 using System.Runtime.CompilerServices;
+using Xunit.Sdk;
 
 internal static class MaxAttemptsHelper
 {
@@ -12,8 +13,8 @@ internal static class MaxAttemptsHelper
     /// </summary>
     /// <param name="testClass">The class defining the test.</param>
     /// <param name="testMethod">The unique name of the test method within <paramref name="testClass"/>.</param>
-    /// <returns>The 0-based attempt number (the value before it was incremented).</returns>
-    public static int GetAndIncrementAttemptNumber(Type testClass, string testMethod)
+    /// <returns>The current (1-based) attempt number.</returns>
+    internal static int GetAndIncrementAttemptNumber(Type testClass, string testMethod)
     {
         StrongBox<int>? attemptNumber;
         lock (AttemptNumber)
@@ -25,6 +26,22 @@ public static int GetAndIncrementAttemptNumber(Type testClass, string testMethod
             }
         }
 
-        return Interlocked.Increment(ref attemptNumber.Value) - 1;
+        return Interlocked.Increment(ref attemptNumber.Value);
+    }
+
+    /// <summary>
+    /// Throws an exception to fail a test if the current attempt number is not the one designated as the passing one.
+    /// </summary>
+    /// <param name="testClass">The test class type. Used for counting attempts.</param>
+    /// <param name="testMethod">The name of the test method, including any theory arguments. Used for counting attempts.</param>
+    /// <param name="passingAttemptNumber">The attempt number which should pass.</param>
+    /// <exception cref="FailException">Thrown when the <paramref name="passingAttemptNumber"/> does not match the current attempt number.</exception>
+    internal static void ThrowUnlessAttemptNumber(Type testClass, string testMethod, int passingAttemptNumber)
+    {
+        int attemptNumber = GetAndIncrementAttemptNumber(testClass, testMethod);
+        if (attemptNumber != passingAttemptNumber)
+        {
+            Assert.Fail($"This test fails on attempt {attemptNumber} (but is expected to pass on attempt {passingAttemptNumber}).");
+        }
     }
 }

test/Xunit.StaFact.Tests/StaFactTests.cs

@@ -44,23 +44,11 @@ public async void AsyncVoid_IsNotSupported()
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 2);
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 1);
 
     [DesktopFact, Trait("TestCategory", "FailureExpected")]
     [UISettings(MaxAttempts = 2)]

test/Xunit.StaFact.Tests/StaTheoryTests.cs

@@ -77,25 +77,13 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o)
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 2);
 
     [DesktopTheory]
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 1);
 
     [DesktopTheory, Trait("TestCategory", "FailureExpected")]
     [InlineData(0)]

test/Xunit.StaFact.Tests/UIFactTests.cs

@@ -147,23 +147,11 @@ await Task.Run(delegate
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 2);
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 1);
 
     [DesktopFact, Trait("TestCategory", "FailureExpected")]
     [UISettings(MaxAttempts = 2)]
@@ -176,13 +164,7 @@ public void FailsAllRetries()
     public class ClassWithDefaultRetryPolicy
     {
         [DesktopFact]
-        public void AutomaticRetryNeeded()
-        {
-            if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 1)
-            {
-                Assert.Fail("The first attempt false, but a second attempt will pass.");
-            }
-        }
+        public void AutomaticRetryNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 2);
 
         [DesktopFact, Trait("TestCategory", "FailureExpected")]
         public void FailsAllRetries()
@@ -192,12 +174,6 @@ public void FailsAllRetries()
 
         [DesktopFact]
         [UISettings(MaxAttempts = 3)]
-        public void SucceedOn3rdAttempt()
-        {
-            if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 2)
-            {
-                Assert.Fail("This attempt should fail except for the 3rd attempt.");
-            }
-        }
+        public void SucceedOn3rdAttempt() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 3);
     }
 }

test/Xunit.StaFact.Tests/UITheoryTests.cs

@@ -151,25 +151,13 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o)
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 2);
 
     [DesktopTheory]
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 1);
 
     [DesktopTheory, Trait("TestCategory", "FailureExpected")]
     [InlineData(0)]

test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsFactTests.cs

@@ -72,23 +72,11 @@ public async Task OperationCanceledException_Thrown()
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 2);
 
     [DesktopFact]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded()
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name) != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded() => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), MethodBase.GetCurrentMethod()!.Name, 1);
 
     [DesktopFact, Trait("TestCategory", "FailureExpected")]
     [UISettings(MaxAttempts = 2)]

test/Xunit.StaFact.Tests/WindowsDesktop/WinFormsTheoryTests.cs

@@ -49,25 +49,13 @@ public void ThreadAffinitizedDataObject(NonSerializableObject o)
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 1)
-        {
-            Assert.Fail("The first attempt false, but a second attempt will pass.");
-        }
-    }
+    public void AutomaticRetryNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 2);
 
     [DesktopTheory]
     [InlineData(0)]
     [InlineData(1)]
     [UISettings(MaxAttempts = 2)]
-    public void AutomaticRetryNotNeeded(int arg)
-    {
-        if (MaxAttemptsHelper.GetAndIncrementAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}") != 0)
-        {
-            Assert.Fail("This test should not have run a second time because the first run was successful.");
-        }
-    }
+    public void AutomaticRetryNotNeeded(int arg) => MaxAttemptsHelper.ThrowUnlessAttemptNumber(this.GetType(), $"{MethodBase.GetCurrentMethod()!.Name}_{arg}", 1);
 
     [DesktopTheory, Trait("TestCategory", "FailureExpected")]
     [InlineData(0)]