test: process to skip tests (#2603)

* remove _disabled suffix from skipped test cases
* remove _disabled from scheme skipped test names
* update CONTRIBUTING.md

CONTRIBUTING.md

@@ -53,7 +53,19 @@ make test
 
 ### Flaky tests
 
-If you see a test being flaky, you should ideally fix it immediately. If that's not feasible, you can disable the test in the test scheme, add a suffix _disabled to the test, so it's clear when looking at the test that it is disabled, and create a GH issue with the issue template flaky test. Disabling the test in the scheme has the advantage that the test report will state "X tests passed, Y tests failed, Z tests skipped".
+If you see a test being flaky, you should ideally fix it immediately. If that's not feasible, you can disable the test in the test scheme by unchecking it in the Test action:
+
+![Disabling test cases via the Xcode scheme](./develop-docs/disabling_tests_xcode_scheme.png)
+
+or by right-clicking it in the Tests Navigator (⌘6):
+
+![Disabling test cases via the Xcode Tests navigator](./develop-docs/disabling_tests_xcode_tests_navigator.png)
+
+Then create a GH issue with the [flaky test issue template](https://github.com/getsentry/sentry-cocoa/issues/new?assignees=&labels=Platform%3A+Cocoa%2CType%3A+Flaky+Test&template=flaky-test.yml). 
+
+Disabling the test in the scheme has the advantage that the test report will state "X tests passed, Y tests failed, Z tests skipped", as well as maintaining a centralized list of skipped tests (look in Sentry.xcscheme) and they will be grayed out when viewing in the Xcode Tests Navigator (⌘6):
+
+![How Xcode displays skipped tests in the Tests Navigator](./develop-docs/xcode_tests_navigator_with_skipped_test.png)
 
 ## Code Formatting
 

Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme

@@ -56,40 +56,40 @@
             </BuildableReference>
             <SkippedTests>
                <Test
-                  Identifier = "SentryCrashIntegrationTests/testStartUpCrash_CallsFlush_disabled()">
+                  Identifier = "SentryCrashIntegrationTests/testStartUpCrash_CallsFlush()">
                </Test>
                <Test
-                  Identifier = "SentryCrashReportStore_Tests/testCrashReportCount1_disabled">
+                  Identifier = "SentryCrashReportStore_Tests/testCrashReportCount1">
                </Test>
                <Test
-                  Identifier = "SentryFileIOTrackingIntegrationTests/test_DataConsistency_readPath_disabled()">
+                  Identifier = "SentryFileIOTrackingIntegrationTests/test_DataConsistency_readPath()">
                </Test>
                <Test
-                  Identifier = "SentryFileIOTrackingIntegrationTests/test_DataConsistency_readUrl_disabled()">
+                  Identifier = "SentryFileIOTrackingIntegrationTests/test_DataConsistency_readUrl()">
                </Test>
                <Test
-                  Identifier = "SentryNSErrorTests/testSerializeWithUnderlyingNSException_disabled()">
+                  Identifier = "SentryNSErrorTests/testSerializeWithUnderlyingNSException()">
                </Test>
                <Test
-                  Identifier = "SentryNetworkTrackerIntegrationTests/testGetRequest_SpanCreatedAndBaggageHeaderAdded_disabled()">
+                  Identifier = "SentryNetworkTrackerIntegrationTests/testGetRequest_SpanCreatedAndBaggageHeaderAdded()">
                </Test>
                <Test
-                  Identifier = "SentryProfilerSwiftTests/testConcurrentSpansWithTimeout_disabled()">
+                  Identifier = "SentryProfilerSwiftTests/testConcurrentSpansWithTimeout()">
                </Test>
                <Test
-                  Identifier = "SentryProfilerSwiftTests/testProfileTimeoutTimer_disabled()">
+                  Identifier = "SentryProfilerSwiftTests/testProfileTimeoutTimer()">
                </Test>
                <Test
                   Identifier = "SentrySDKIntegrationTestsBase">
                </Test>
                <Test
-                  Identifier = "SentrySessionGeneratorTests/testSendSessions_disabled()">
+                  Identifier = "SentrySessionGeneratorTests/testSendSessions()">
                </Test>
                <Test
-                  Identifier = "SentryStacktraceBuilderTests/testAsyncStacktraces_disabled()">
+                  Identifier = "SentryStacktraceBuilderTests/testAsyncStacktraces()">
                </Test>
                <Test
-                  Identifier = "SentrySubClassFinderTests/testActOnSubclassesOfViewController_disabled()">
+                  Identifier = "SentrySubClassFinderTests/testActOnSubclassesOfViewController()">
                </Test>
                <Test
                   Identifier = "SentryUIApplicationTests/test_applicationWithScenes()">

Tests/SentryTests/Integrations/Performance/IO/SentryFileIOTrackingIntegrationTests.swift

@@ -191,7 +191,7 @@ class SentryFileIOTrackingIntegrationTests: XCTestCase {
         ?? bundle.path(forResource: "fatal-error-binary-images-message2", ofType: "json")
     }
 
-    func test_DataConsistency_readUrl_disabled() {
+    func test_DataConsistency_readUrl() {
         SentrySDK.start(options: fixture.getOptions())
 
         let randomValue = UUID().uuidString
@@ -205,7 +205,7 @@ class SentryFileIOTrackingIntegrationTests: XCTestCase {
         XCTAssertEqual(randomValue, readValue)
     }
 
-    func test_DataConsistency_readPath_disabled() {
+    func test_DataConsistency_readPath() {
         SentrySDK.start(options: fixture.getOptions())
 
         let randomValue = UUID().uuidString

Tests/SentryTests/Integrations/Performance/Network/SentryNetworkTrackerIntegrationTests.swift

@@ -151,7 +151,7 @@ class SentryNetworkTrackerIntegrationTests: XCTestCase {
         XCTAssertEqual(1, breadcrumbs?.count)
     }
 
-    func testGetRequest_SpanCreatedAndBaggageHeaderAdded_disabled() {
+    func testGetRequest_SpanCreatedAndBaggageHeaderAdded() {
         startSDK()
         let transaction = SentrySDK.startTransaction(name: "Test Transaction", operation: "TEST", bindToScope: true) as! SentryTracer
         let expect = expectation(description: "Request completed")

Tests/SentryTests/Integrations/Performance/SentrySubClassFinderTests.swift

@@ -38,7 +38,7 @@ class SentrySubClassFinderTests: XCTestCase {
         fixture = Fixture()
     }
 
-    func testActOnSubclassesOfViewController_disabled() {
+    func testActOnSubclassesOfViewController() {
         assertActOnSubclassesOfViewController(expected: [FirstViewController.self, SecondViewController.self, ViewControllerNumberThree.self, VCAnyNaming.self])
     }
 

Tests/SentryTests/Integrations/SentryCrash/SentryCrashIntegrationTests.swift

@@ -261,8 +261,7 @@ class SentryCrashIntegrationTests: NotificationCenterTestCase {
         assertLocaleOnHub(locale: Locale.autoupdatingCurrent.identifier, hub: hub)
     }
 
-    // !!!: Disabled until flakiness can be fixed
-    func testStartUpCrash_CallsFlush_disabled() throws {
+    func testStartUpCrash_CallsFlush() throws {
         let (sut, hub) = givenSutWithGlobalHubAndCrashWrapper()
         sut.install(with: Options())
 

Tests/SentryTests/Integrations/Session/SentrySessionGeneratorTests.swift

@@ -58,7 +58,7 @@ class SentrySessionGeneratorTests: NotificationCenterTestCase {
     /**
      * Disabled on purpose. This test just sends sessions to Sentry, but doesn't verify that they arrive there properly.
      */
-    func testSendSessions_disabled() {
+    func testSendSessions() {
         sendSessions(amount: Sessions(healthy: 10, errored: 10, crashed: 3, oom: 1, abnormal: 1))
     }
 

Tests/SentryTests/Integrations/ViewHierarchy/SentryViewHierarchyIntegrationTests.swift

@@ -33,7 +33,7 @@ class SentryViewHierarchyIntegrationTests: XCTestCase {
         clearTestState()
     }
 
-    func test_attachViewHierarchy_disabled() {
+    func test_attachViewHierarchy() {
         SentrySDK.start { $0.attachViewHierarchy = false }
         XCTAssertEqual(SentrySDK.currentHub().getClient()?.attachmentProcessors.count, 0)
         XCTAssertFalse(sentrycrash_hasSaveViewHierarchyCallback())

Tests/SentryTests/Profiling/SentryProfilerSwiftTests.swift

@@ -80,7 +80,7 @@ class SentryProfilerSwiftTests: XCTestCase {
     ///    transaction B                                           |-------|
     ///    profiler B                                              |-------|  <- normal finish
     ///   ```
-    func testConcurrentSpansWithTimeout_disabled() {
+    func testConcurrentSpansWithTimeout() {
         let options = fixture.options
         options.profilesSampleRate = 1.0
         options.tracesSampleRate = 1.0
@@ -117,7 +117,7 @@ class SentryProfilerSwiftTests: XCTestCase {
         }
     }
 
-    func testProfileTimeoutTimer_disabled() {
+    func testProfileTimeoutTimer() {
         fixture.options.profilesSampleRate = 1.0
         fixture.options.tracesSampleRate = 1.0
         performTest(shouldTimeOut: true)

Tests/SentryTests/Protocol/SentryNSErrorTests.swift

@@ -27,7 +27,7 @@ class SentryNSErrorTests: XCTestCase {
         XCTAssertEqual(actualUnderlyingError.domain, inputUnderlyingError.domain)
     }
 
-    func testSerializeWithUnderlyingNSException_disabled() {
+    func testSerializeWithUnderlyingNSException() {
         let inputExceptionName = NSExceptionName.decimalNumberDivideByZeroException
         let inputExceptionReason = "test exception reason"
         let inputUnderlyingException = NSException(name: inputExceptionName, reason: inputExceptionReason, userInfo: ["some userinfo key": "some userinfo value"])

Tests/SentryTests/SentryCrash/SentryCrashReportStore_Tests.m

@@ -141,7 +141,7 @@ - (void)testReportStorePathExists
     XCTAssertTrue([[NSFileManager defaultManager] fileExistsAtPath:self.reportStorePath]);
 }
 
-- (void)testCrashReportCount1_disabled
+- (void)testCrashReportCount1
 {
     [self prepareReportStoreWithPathEnd:@"testCrashReportCount1"];
     NSString *reportContents = @"Testing";

Tests/SentryTests/SentryCrash/SentryStacktraceBuilderTests.swift

@@ -67,7 +67,7 @@ class SentryStacktraceBuilderTests: XCTestCase {
         XCTAssertTrue(filteredFrames.count == 1, "The frames must be ordered from caller to callee, or oldest to youngest.")
     }
 
-    func testAsyncStacktraces_disabled() throws {
+    func testAsyncStacktraces() throws {
         SentrySDK.start { options in
             options.dsn = TestConstants.dsnAsString(username: "SentryStacktraceBuilderTests")
             options.stitchAsyncCode = true