Skip to content

Commit

Permalink
[xharness] Enable monotouch-test/.NET by default, but ignore all vari…
Browse files Browse the repository at this point in the history
…ations. (#9526)

* [xharness] Enable monotouch-test/.NET by default, but ignore all variations.

The default (Debug) configuration is green now, but the rest of the variations aren't yet.

* [tests] Ignore LocaleTest.InitRegionInfo in .NET because it needs globalization data.

Ref: #8906

I can't reproduce locally, because it's ignored for me:

    [INCONCLUSIVE] InitRegionInfo : You can construct locale without countries
        at MonoTouchFixtures.Foundation.LocaleTest.InitRegionInfo() in [...]/xamarin-macios/tests/monotouch-test/Foundation/LocaleTest.cs:line 47

but this should fix this monotouch-test test failure (I'm guessing that 'IV'
stands for 'Invariant', which is all the current .NET version ships with):

    MonoTouchFixtures.Foundation.LocaleTest
        [FAIL] InitRegionInfo :   Name
            String lengths are both 2. Strings differ at index 0.
            Expected: "US"
            But was:  "IV"
            -----------^
                 :    at MonoTouchFixtures.Foundation.LocaleTest.InitRegionInfo() in /Users/builder/jenkins/workspace/xamarin-macios-pr-builder/tests/monotouch-test/Foundation/LocaleTest.cs:line 49
  • Loading branch information
rolfbjarne authored Aug 27, 2020
1 parent d5ddc92 commit e77a737
Show file tree
Hide file tree
Showing 3 changed files with 8 additions and 1 deletion.
3 changes: 3 additions & 0 deletions tests/monotouch-test/Foundation/LocaleTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ public void FromLocaleIdentifier ()
Assert.That (NSLocale.FromLocaleIdentifier (ident).Identifier, Is.EqualTo (ident), "FromLocaleIdentifier");
}

#if NET
[Ignore ("No globalization data yet - https://github.com/xamarin/xamarin-macios/issues/8906")]
#endif
[Test]
public void InitRegionInfo ()
{
Expand Down
2 changes: 1 addition & 1 deletion tests/xharness/Harness.cs
Original file line number Diff line number Diff line change
Expand Up @@ -391,7 +391,7 @@ void AutoConfigureIOS ()
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "interdependent-binding-projects", "dotnet", "iOS", "interdependent-binding-projects.csproj"))) { Name = "interdependent-binding-projects", IsDotNetProject = true, SkipiOSVariation = false, SkiptvOSVariation = true, SkipwatchOSVariation = true, SkipTodayExtensionVariation = true, SkipDeviceVariations = true, SkipiOS32Variation = true, });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "introspection", "iOS", "introspection-ios.csproj"))) { Name = "introspection" });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "introspection", "iOS", "introspection-ios-dotnet.csproj"))) { Name = "introspection", IsDotNetProject = true, SkipiOSVariation = false, SkiptvOSVariation = false, SkipwatchOSVariation = true, SkipTodayExtensionVariation = true, SkipDeviceVariations = true, SkipiOS32Variation = true, });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "monotouch-test", "dotnet", "iOS", "monotouch-test.csproj"))) { Name = "monotouch-test", IsDotNetProject = true, SkipiOSVariation = false, SkiptvOSVariation = true, SkipwatchOSVariation = true, SkipTodayExtensionVariation = true, SkipDeviceVariations = true, SkipiOS32Variation = true, Ignore = true, });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "monotouch-test", "dotnet", "iOS", "monotouch-test.csproj"))) { Name = "monotouch-test", IsDotNetProject = true, SkipiOSVariation = false, SkiptvOSVariation = true, SkipwatchOSVariation = true, SkipTodayExtensionVariation = true, SkipDeviceVariations = true, SkipiOS32Variation = true, });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "dont link", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" } });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "dont link", "dotnet", "iOS", "dont link.csproj"))) { Configurations = new string [] { "Debug", "Release" }, IsDotNetProject = true, SkipiOSVariation = false, SkiptvOSVariation = true, SkipwatchOSVariation = true, SkipTodayExtensionVariation = true, SkipDeviceVariations = true, SkipiOS32Variation = true });
IOSTestProjects.Add (new iOSTestProject (Path.GetFullPath (Path.Combine (RootDirectory, "linker", "ios", "link all", "link all.csproj"))) { Configurations = new string [] { "Debug", "Release" } });
Expand Down
4 changes: 4 additions & 0 deletions tests/xharness/Jenkins/TestVariationsFactory.cs
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ IEnumerable<TestData> GetTestData (RunTestTask test)

switch (test.TestName) {
case "monotouch-test":
if (test.TestProject.IsDotNetProject)
ignore = true;
if (supports_dynamic_registrar_on_device)
yield return new TestData { Variation = "Debug (dynamic registrar)", MTouchExtraArgs = "--registrar:dynamic", Debug = true, Profiling = false, Ignored = ignore };
yield return new TestData { Variation = "Release (all optimizations)", MTouchExtraArgs = "--registrar:static --optimize:all", Debug = false, Profiling = false, Defines = "OPTIMIZEALL", Ignored = ignore };
Expand Down Expand Up @@ -87,6 +89,8 @@ IEnumerable<TestData> GetTestData (RunTestTask test)
case "iPhoneSimulator":
switch (test.TestName) {
case "monotouch-test":
if (test.TestProject.IsDotNetProject)
ignore = true;
// The default is to run monotouch-test with the dynamic registrar (in the simulator), so that's already covered
yield return new TestData { Variation = "Debug (LinkSdk)", Debug = true, Profiling = false, LinkMode = "LinkSdk", Ignored = ignore };
yield return new TestData { Variation = "Debug (static registrar)", MTouchExtraArgs = "--registrar:static", Debug = true, Profiling = false, Undefines = "DYNAMIC_REGISTRAR", Ignored = ignore };
Expand Down

5 comments on commit e77a737

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔥 Tests failed catastrophically on VSTS: device tests iOS (Cambridge) 🔥

Not enough free space in the host.

Pipeline on Agent

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔥 Tests failed catastrophically on VSTS: device tests iOS (DDFun) 🔥

Not enough free space in the host.

Pipeline on Agent

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🔥 Tests failed catastrophically on VSTS: device tests tvOS (DDFun) 🔥

Not enough free space in the host.

Pipeline on Agent

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

❌ Device tests failed on VSTS: device tests iOS32b (Cambridge) ❌

Device tests failed on VSTS: device tests iOS32b (Cambridge).

Test results

119 tests failed, 8 tests' device not found, 34 tests passed.

Failed tests

  • monotouch-test/iOS Unified 32-bits - device/Debug: Crashed
  • monotouch-test/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • monotouch-test/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • monotouch-test/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • monotouch-test/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • monotouch-test/iOS Unified 32-bits - device/Release: Crashed
  • monotouch-test/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • monotouch-test/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • monotouch-test/iOS Unified 32-bits - device/Release (all optimizations): Crashed
  • monotouch-test/iOS Unified 32-bits - device/Debug (all optimizations): Crashed
  • monotouch-test/iOS Unified 32-bits - device/Debug: SGenConc: Crashed
  • framework-test/iOS Unified 32-bits - device/Debug: Crashed
  • fsharp/iOS Unified 32-bits - device/Debug: Crashed
  • interdependent-binding-projects/iOS Unified 32-bits - device/Debug: Crashed
  • dont link/iOS Unified 32-bits - device/Release: Crashed
  • link all/iOS Unified 32-bits - device/Debug: Crashed
  • link sdk/iOS Unified 32-bits - device/Debug: Crashed
  • link sdk/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • link sdk/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed (HE0052: Failed to stop session: Read Error (error: 0xe8000004))
  • link sdk/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • link sdk/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • link sdk/iOS Unified 32-bits - device/Release: Crashed
  • link sdk/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • link sdk/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • mono-native-compat/iOS Unified 32-bits - device/Debug: Crashed
  • mono-native-compat/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • mono-native-compat/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • mono-native-compat/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Failed (Install failed, exit code: 1.)
  • mono-native-compat/iOS Unified 32-bits - device/Release: Crashed
  • mono-native-compat/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • mono-native-compat/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/Debug: Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/Release: Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [NUnit] Mono BCL tests group 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/Debug: Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/Release: Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [NUnit] Mono BCL tests group 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/Debug: Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/Release: Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [xUnit] Mono BCL tests group 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/Debug: Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/Release: Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [xUnit] Mono BCL tests group 4/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/Debug: Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • mscorlib Part 1/iOS Unified 32-bits - device/Release: BuildFailure Known issue: Undefined symbol ___multi3 on Release Mode.
  • mscorlib Part 1/iOS Unified 32-bits - device/Release: UseThumb: BuildFailure Known issue: Undefined symbol ___multi3 on Release Mode.
  • mscorlib Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): BuildFailure Known issue: Undefined symbol ___multi3 on Release Mode.
  • mscorlib Part 1/iOS Unified 32-bits - device/Debug: SGenConc: Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/Debug: Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/Release: Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • mscorlib Part 2/iOS Unified 32-bits - device/Debug: SGenConc: Crashed (HE0052: Failed to stop session: Read Error (error: 0xe8000004))
  • mscorlib Part 3/iOS Unified 32-bits - device/Debug: Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/Release: Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • mscorlib Part 3/iOS Unified 32-bits - device/Debug: SGenConc: Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/Debug: Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/Release: Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [xUnit] Mono SystemCoreXunit Part 1/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/Debug: Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed (HE0052: Failed to stop session: Read Error (error: 0xe8000004))
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/Release: Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/Release: UseThumb: Crashed
  • [xUnit] Mono SystemCoreXunit Part 2/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/Debug: Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/Release: Crashed
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/Release: UseThumb: Crashed (HE0052: Failed to stop session: Read Error (error: 0xe8000004))
  • [xUnit] Mono SystemXunit/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (release): Crashed
  • [xUnit] Mono BCL tests group 5/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug): Crashed
  • [xUnit] Mono BCL tests group 5/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug): Crashed
  • [xUnit] Mono BCL tests group 5/iOS Unified 32-bits - device/AssemblyBuildTarget: dylib (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 5/iOS Unified 32-bits - device/AssemblyBuildTarget: SDK framework (debug, profiling): Crashed
  • [xUnit] Mono BCL tests group 5/iOS Unified 32-bits - device/Release: Crashed

Pipeline on Agent XI-W-BULBASAUR

@xamarin-release-manager
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Build was (probably) aborted

🔥 Jenkins job (on internal Jenkins) failed in stage(s) 'Test run' 🔥 : org.jenkinsci.plugins.workflow.steps.FlowInterruptedException

Provisioning succeeded
Build succeeded
✅ Packages built successfully

View packages

API Diff (from stable)
API Diff (from PR only) (no change)
Generator Diff (no change)
🔥 Test run failed 🔥

Test results

Test run in progress: InProgress, Waiting: 1, Building: 2, BuildQueued: 58, Built: 96, RunQueued: 14, Succeeded: 21, Ignored: 1028

Please sign in to comment.