xcodebuild_runner_test.py

#!/usr/bin/env vpython
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Unittests for xcodebuild_runner.py."""

import logging
import mock
import os
import unittest

import iossim_util
import result_sink_util
import test_apps
from test_result_util import ResultCollection, TestResult, TestStatus
import test_runner
import test_runner_test
import xcode_log_parser
import xcodebuild_runner


_ROOT_FOLDER_PATH = 'root/folder'
_XCODE_BUILD_VERSION = '10B61'
_DESTINATION = 'A4E66321-177A-450A-9BA1-488D85B7278E'
_OUT_DIR = 'out/dir'
_XTEST_RUN = '/tmp/temp_file.xctestrun'
_EGTESTS_APP_PATH = '%s/any_egtests.app' % _ROOT_FOLDER_PATH
_FLAKY_EGTEST_APP_PATH = 'path/to/ios_chrome_flaky_eg2test_module.app'


class XCodebuildRunnerTest(test_runner_test.TestCase):
  """Test case to test xcodebuild_runner."""

  def setUp(self):
    super(XCodebuildRunnerTest, self).setUp()
    self.mock(os.path, 'exists', lambda _: True)
    self.mock(xcode_log_parser,
              'get_parser', lambda: xcode_log_parser.Xcode11LogParser())
    self.mock(os, 'listdir', lambda _: ['any_egtests.xctest'])
    self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False)
    self.mock(result_sink_util.ResultSinkClient,
              'post', lambda *args, **kwargs: None)
    self.mock(
        test_apps.GTestsApp,
        'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2'])
    self.mock(test_apps.EgtestsApp,
              'fill_xctest_run', lambda _1, _2: 'xctestrun')
    self.mock(iossim_util, 'get_simulator', lambda _1, _2: 'sim-UUID')
    self.mock(test_apps, 'get_bundle_id', lambda _: "fake-bundle-id")
    self.mock(test_apps, 'is_running_rosetta', lambda: False)
    self.mock(test_apps.plistlib, 'writePlist', lambda _1, _2: '')
    self.mock(test_runner.SimulatorTestRunner, 'tear_down', lambda _: None)
    self.mock(test_runner.DeviceTestRunner, 'tear_down', lambda _: None)
    self.mock(xcodebuild_runner.subprocess,
              'Popen', lambda cmd, env, stdout, stderr: 'fake-out')
    self.mock(test_runner, 'print_process_output', lambda _: [])

  def tearDown(self):
    super(XCodebuildRunnerTest, self).tearDown()

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def testLaunchCommand_restartCrashed1stAttempt(self, mock_collect_results):
    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
    crashed_collection = ResultCollection()
    crashed_collection.crashed = True
    mock_collect_results.side_effect = [
        crashed_collection,
        ResultCollection(test_results=[
            TestResult('Class1/passedTest1', TestStatus.PASS),
            TestResult('Class1/passedTest2', TestStatus.PASS)
        ])
    ]
    launch_command = xcodebuild_runner.LaunchCommand(
        egtests, _DESTINATION, shards=1, retries=3)
    overall_result = launch_command.launch()
    self.assertFalse(overall_result.crashed)
    self.assertEqual(len(overall_result.all_test_names()), 2)
    self.assertEqual(overall_result.expected_tests(),
                     set(['Class1/passedTest1', 'Class1/passedTest2']))

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def testLaunchCommand_notRestartPassedTest(self, mock_collect_results):
    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
    collection = ResultCollection(test_results=[
        TestResult('Class1/passedTest1', TestStatus.PASS),
        TestResult('Class1/passedTest2', TestStatus.PASS)
    ])
    mock_collect_results.side_effect = [collection]
    launch_command = xcodebuild_runner.LaunchCommand(
        egtests, _DESTINATION, shards=1, retries=3)
    launch_command.launch()
    xcodebuild_runner.LaunchCommand(egtests, _DESTINATION, shards=1, retries=3)
    self.assertEqual(1, len(mock_collect_results.mock_calls))

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def test_launch_command_restart_failed_attempt(self, mock_collect_results):
    egtests = test_apps.EgtestsApp(_EGTESTS_APP_PATH)
    mock_collect_results.side_effect = [
        ResultCollection(test_results=[
            TestResult('Class1/passedTest1', TestStatus.FAIL),
            TestResult('Class1/passedTest2', TestStatus.FAIL)
        ]),
        ResultCollection(test_results=[
            TestResult('Class1/passedTest1', TestStatus.PASS),
            TestResult('Class1/passedTest2', TestStatus.PASS)
        ])
    ]
    launch_command = xcodebuild_runner.LaunchCommand(
        egtests, _DESTINATION, shards=1, retries=3)
    overall_result = launch_command.launch()
    self.assertEqual(len(overall_result.all_test_names()), 2)
    self.assertEqual(overall_result.expected_tests(),
                     set(['Class1/passedTest1', 'Class1/passedTest2']))

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def test_launch_command_not_restart_crashed_attempt(self,
                                                      mock_collect_results):
    """Crashed first attempt of runtime select test suite won't be retried."""
    egtests = test_apps.EgtestsApp(_FLAKY_EGTEST_APP_PATH)
    crashed_collection = ResultCollection()
    crashed_collection.crashed = True
    mock_collect_results.return_value = crashed_collection
    launch_command = xcodebuild_runner.LaunchCommand(
        egtests, _DESTINATION, shards=1, retries=3)
    overall_result = launch_command.launch()
    self.assertEqual(len(overall_result.all_test_names()), 0)
    self.assertEqual(overall_result.expected_tests(), set([]))
    self.assertTrue(overall_result.crashed)


class DeviceXcodeTestRunnerTest(test_runner_test.TestCase):
  """Test case to test xcodebuild_runner.DeviceXcodeTestRunner."""

  def setUp(self):
    super(DeviceXcodeTestRunnerTest, self).setUp()
    self.mock(os.path, 'exists', lambda _: True)
    self.mock(test_runner, 'get_current_xcode_info', lambda: {
        'version': 'test version', 'build': 'test build', 'path': 'test/path'})
    self.mock(os.path, 'abspath', lambda path: '/abs/path/to/%s' % path)

    self.mock(result_sink_util.ResultSinkClient,
              'post', lambda *args, **kwargs: None)
    self.mock(test_runner.subprocess, 'check_output', lambda _: b'fake-output')
    self.mock(test_runner.subprocess, 'check_call', lambda _: b'fake-out')
    self.mock(test_runner.subprocess,
              'Popen', lambda cmd, env, stdout, stderr: 'fake-out')
    self.mock(test_runner.TestRunner, 'set_sigterm_handler',
              lambda self, handler: 0)
    self.mock(os, 'listdir', lambda _: [])
    self.mock(xcodebuild_runner.subprocess,
              'Popen', lambda cmd, env, stdout, stderr: 'fake-out')
    self.mock(test_runner, 'print_process_output', lambda _: [])
    self.mock(test_runner.TestRunner, 'start_proc', lambda self, cmd: 0)
    self.mock(test_runner.DeviceTestRunner, 'get_installed_packages',
              lambda self: [])
    self.mock(test_runner.DeviceTestRunner, 'wipe_derived_data', lambda _: None)
    self.mock(test_runner.TestRunner, 'retrieve_derived_data', lambda _: None)
    self.mock(test_runner.TestRunner, 'process_xcresult_dir', lambda _: None)
    self.mock(xcode_log_parser,
              'get_parser', lambda: xcode_log_parser.Xcode11LogParser())
    self.mock(test_apps.EgtestsApp,
              'fill_xctest_run', lambda _1, _2: 'xctestrun')
    self.mock(
        test_apps.GTestsApp,
        'get_all_tests', lambda _: ['Class1/passedTest1', 'Class1/passedTest2'])
    self.mock(iossim_util, 'is_device_with_udid_simulator', lambda _: False)

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def test_launch(self, mock_result):
    """Tests launch method in DeviceXcodeTestRunner"""
    tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path",
                                                 "fake-host-app-path",
                                                 "fake-out-dir")
    mock_result.return_value = ResultCollection(test_results=[
        TestResult('Class1/passedTest1', TestStatus.PASS),
        TestResult('Class1/passedTest2', TestStatus.PASS)
    ])
    self.assertTrue(tr.launch())
    self.assertEqual(len(tr.test_results['tests']), 2)

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def test_unexpected_skipped_crash_reported(self, mock_result):
    """Tests launch method in DeviceXcodeTestRunner"""
    tr = xcodebuild_runner.DeviceXcodeTestRunner("fake-app-path",
                                                 "fake-host-app-path",
                                                 "fake-out-dir")
    crashed_collection = ResultCollection(
        test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)])
    crashed_collection.crashed = True
    mock_result.return_value = crashed_collection
    self.assertFalse(tr.launch())
    self.assertEqual(len(tr.test_results['tests']), 2)
    tests = tr.test_results['tests']
    self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
    self.assertEqual(tests['Class1/passedTest2']['actual'], 'SKIP')
    self.assertEqual(tests['Class1/passedTest2']['expected'], 'PASS')

  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  def test_unexpected_skipped_not_reported(self, mock_result):
    """Unexpected skip not reported for these selecting tests at runtime."""
    crashed_collection = ResultCollection(
        test_results=[TestResult('Class1/passedTest1', TestStatus.PASS)])
    crashed_collection.crashed = True
    mock_result.return_value = crashed_collection
    tr = xcodebuild_runner.DeviceXcodeTestRunner(_FLAKY_EGTEST_APP_PATH,
                                                 "fake-host-app-path",
                                                 "fake-out-dir")
    self.assertFalse(tr.launch())
    self.assertEqual(len(tr.test_results['tests']), 1)
    tests = tr.test_results['tests']
    self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
    # Class1/passedTest2 doesn't appear in test results.

  @mock.patch('xcodebuild_runner.isinstance', return_value=True)
  @mock.patch('xcode_log_parser.Xcode11LogParser.collect_test_results')
  @mock.patch('test_apps.EgtestsApp', autospec=True)
  def test_disabled_reported(self, mock_test_app, mock_result, _):
    """Tests launch method in DeviceXcodeTestRunner"""
    test_app = mock_test_app.return_value
    test_app.test_app_path = _EGTESTS_APP_PATH
    test_app.disabled_tests = ['Class2/disabled_test3']
    test_app.get_all_tests.return_value = [
        'Class1/passedTest1', 'Class1/passedTest2'
    ]
    mock_result.return_value = ResultCollection(test_results=[
        TestResult('Class1/passedTest1', TestStatus.PASS),
        TestResult('Class1/passedTest2', TestStatus.PASS)
    ])
    tr = xcodebuild_runner.DeviceXcodeTestRunner(
        "fake-app-path", "fake-host-app-path", "fake-out-dir")
    self.assertTrue(tr.launch())
    self.assertEqual(len(tr.test_results['tests']), 3)
    tests = tr.test_results['tests']
    self.assertEqual(tests['Class1/passedTest1']['actual'], 'PASS')
    self.assertEqual(tests['Class1/passedTest2']['actual'], 'PASS')
    self.assertEqual(tests['Class2/disabled_test3']['actual'], 'SKIP')
    self.assertEqual(tests['Class2/disabled_test3']['expected'], 'SKIP')

  def test_tear_down(self):
    tr = xcodebuild_runner.DeviceXcodeTestRunner(
        "fake-app-path", "fake-host-app-path", "fake-out-dir")
    tr.tear_down()


if __name__ == '__main__':
  logging.basicConfig(
      format='[%(asctime)s:%(levelname)s] %(message)s',
      level=logging.DEBUG,
      datefmt='%I:%M:%S')
  unittest.main()