Merge branch 'develop' into missing-phpunit

CHANGELOG.md

@@ -1,6 +1,11 @@
 Magento Functional Testing Framework Changelog
 ================================================
 
+2.3.11
+-----
+### Fixes
+* `mftf run:failed` now correctly regenerates tests that are in suites that were parallelized (`suite` => `suite_0`, `suite_1`)
+
 2.3.10
 -----
 ### Enhancements

bin/mftf

@@ -29,7 +29,7 @@ try {
 try {
     $application = new Symfony\Component\Console\Application();
     $application->setName('Magento Functional Testing Framework CLI');
-    $application->setVersion('2.3.10');
+    $application->setVersion('2.3.11');
     /** @var \Magento\FunctionalTestingFramework\Console\CommandListInterface $commandList */
     $commandList = new \Magento\FunctionalTestingFramework\Console\CommandList;
     foreach ($commandList->getCommands() as $command) {

composer.json

@@ -2,7 +2,7 @@
     "name": "magento/magento2-functional-testing-framework",
     "description": "Magento2 Functional Testing Framework",
     "type": "library",
-    "version": "2.3.10",
+    "version": "2.3.11",
     "license": "AGPL-3.0",
     "keywords": ["magento", "automation", "functional", "testing"],
     "config": {

dev/tests/_bootstrap.php

@@ -82,7 +82,8 @@
 
 $paths = [
     $suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuite.xml',
-    $suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuiteHooks.xml'
+    $suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuiteHooks.xml',
+    $suiteDirectory . DIRECTORY_SEPARATOR . 'functionalSuiteExtends.xml'
 ];
 
 // create and return the iterator for these file paths

dev/tests/verification/Resources/ExtendedTestInSuiteChildTestCest.txt

@@ -0,0 +1,63 @@
+<?php
+namespace Magento\AcceptanceTest\_suiteExtends\Backend;
+
+use Magento\FunctionalTestingFramework\AcceptanceTester;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\CredentialStore;
+use Magento\FunctionalTestingFramework\DataGenerator\Handlers\PersistedObjectHandler;
+use \Codeception\Util\Locator;
+use Yandex\Allure\Adapter\Annotation\Features;
+use Yandex\Allure\Adapter\Annotation\Stories;
+use Yandex\Allure\Adapter\Annotation\Title;
+use Yandex\Allure\Adapter\Annotation\Description;
+use Yandex\Allure\Adapter\Annotation\Parameter;
+use Yandex\Allure\Adapter\Annotation\Severity;
+use Yandex\Allure\Adapter\Model\SeverityLevel;
+use Yandex\Allure\Adapter\Annotation\TestCaseId;
+
+/**
+ * @Title("[NO TESTCASEID]: ExtendedTestInSuiteChildTest")
+ * @group ExtendedTestInSuiteChildTest
+ */
+class ExtendedTestInSuiteChildTestCest
+{
+	/**
+	  * @param AcceptanceTester $I
+	  * @throws \Exception
+	  */
+	public function _before(AcceptanceTester $I)
+	{
+		$I->amOnPage("/beforeUrl");
+	}
+
+	/**
+	  * @param AcceptanceTester $I
+	  * @throws \Exception
+	  */
+	public function _after(AcceptanceTester $I)
+	{
+		$I->amOnPage("/afterUrl");
+	}
+
+	/**
+	  * @param AcceptanceTester $I
+	  * @throws \Exception
+	  */
+	public function _failed(AcceptanceTester $I)
+	{
+		$I->saveScreenshot();
+	}
+
+	/**
+	 * @Severity(level = SeverityLevel::TRIVIAL)
+	 * @Features({"TestModule"})
+	 * @Stories({"ExtendedTestInSuiteChildTest"})
+	 * @Parameter(name = "AcceptanceTester", value="$I")
+	 * @param AcceptanceTester $I
+	 * @return void
+	 * @throws \Exception
+	 */
+	public function ExtendedTestInSuiteChildTest(AcceptanceTester $I)
+	{
+		$I->comment("Different Input");
+	}
+}

dev/tests/verification/TestModule/Test/ExtendedFunctionalTest.xml

@@ -158,4 +158,32 @@
         </annotations>
         <comment userInput="child" stepKey="replaceMe"/>
     </test>
+
+    <test name="ExtendedTestInSuiteParentTest">
+        <annotations>
+            <severity value="AVERAGE"/>
+            <title value="ExtendedTestInSuiteParentTest"/>
+            <group value="ExtendedTestInSuite"/>
+            <features value="ExtendedTestInSuite"/>
+            <stories value="ExtendedTestInSuite"/>
+        </annotations>
+        <before>
+            <amOnPage url="/beforeUrl" stepKey="beforeAmOnPageKey"/>
+        </before>
+        <after>
+            <amOnPage url="/afterUrl" stepKey="afterAmOnPageKey"/>
+        </after>
+        <comment stepKey="basicCommentWithNoData" userInput="Parent Comment"/>
+    </test>
+
+    <test name="ExtendedTestInSuiteChildTest" extends="ExtendedTestInSuiteParentTest">
+        <annotations>
+            <severity value="MINOR"/>
+            <title value="ExtendedTestInSuiteChildTest"/>
+            <group value="ExtendedTestInSuiteChildTest"/>
+            <features value="ExtendedTestInSuiteChildTest"/>
+            <stories value="ExtendedTestInSuiteChildTest"/>
+        </annotations>
+        <comment stepKey="basicCommentWithNoData" userInput="Different Input"/>
+    </test>
 </tests>

dev/tests/verification/Tests/SuiteGenerationTest.php

@@ -285,6 +285,45 @@ public function testSuiteGenerationSingleRun()
         $this->assertEquals($expectedManifest, file_get_contents(self::getManifestFilePath()));
     }
 
+    /**
+     * Test extends tests generation in a suite
+     */
+    public function testSuiteGenerationWithExtends()
+    {
+        $groupName = 'suiteExtends';
+
+        $expectedContents = [
+            'ExtendedTestInSuiteChildTestCest.php'
+        ];
+
+        // Generate the Suite
+        SuiteGenerator::getInstance()->generateSuite($groupName);
+
+        // Validate log message and add group name for later deletion
+        TestLoggingUtil::getInstance()->validateMockLogStatement(
+            'info',
+            "suite generated",
+            ['suite' => $groupName, 'relative_path' => "_generated" . DIRECTORY_SEPARATOR . $groupName]
+        );
+        self::$TEST_GROUPS[] = $groupName;
+
+        // Validate Yaml file updated
+        $yml = Yaml::parse(file_get_contents(self::CONFIG_YML_FILE));
+        $this->assertArrayHasKey($groupName, $yml['groups']);
+
+        $suiteResultBaseDir = self::GENERATE_RESULT_DIR .
+            DIRECTORY_SEPARATOR .
+            $groupName .
+            DIRECTORY_SEPARATOR;
+
+        // Validate tests have been generated
+        $dirContents = array_diff(scandir($suiteResultBaseDir), ['..', '.']);
+
+        foreach ($expectedContents as $expectedFile) {
+            $this->assertTrue(in_array($expectedFile, $dirContents));
+        }
+    }
+
     /**
      * revert any changes made to config.yml
      * remove _generated directory

dev/tests/verification/_suite/functionalSuiteExtends.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ /**
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-->
+
+<suites xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../../../src/Magento/FunctionalTestingFramework/Suite/etc/suiteSchema.xsd">
+    <suite name="suiteExtends">
+        <include>
+            <group name="ExtendedTestInSuiteChildTest"/>
+        </include>
+    </suite>
+</suites>

etc/config/functional.suite.dist.yml

@@ -14,12 +14,6 @@ modules:
         - \Magento\FunctionalTestingFramework\Module\MagentoWebDriver
         - \Magento\FunctionalTestingFramework\Helper\Acceptance
         - \Magento\FunctionalTestingFramework\Helper\MagentoFakerData
-        - \Magento\FunctionalTestingFramework\Module\MagentoRestDriver:
-            url: "%MAGENTO_BASE_URL%/rest/default/V1/"
-            username: "%MAGENTO_ADMIN_USERNAME%"
-            password: "%MAGENTO_ADMIN_PASSWORD%"
-            depends: PhpBrowser
-            part: Json
         - \Magento\FunctionalTestingFramework\Module\MagentoSequence
         - \Magento\FunctionalTestingFramework\Module\MagentoAssert
         - Asserts

src/Magento/FunctionalTestingFramework/Allure/Adapter/MagentoAllureAdapter.php

@@ -5,10 +5,11 @@
  */
 namespace Magento\FunctionalTestingFramework\Allure\Adapter;
 
-use Magento\FunctionalTestingFramework\Data\Argument\Interpreter\NullType;
 use Magento\FunctionalTestingFramework\Suite\Handlers\SuiteObjectHandler;
 use Yandex\Allure\Adapter\AllureAdapter;
 use Yandex\Allure\Adapter\Event\StepStartedEvent;
+use Yandex\Allure\Adapter\Event\StepFinishedEvent;
+use Yandex\Allure\Adapter\Event\StepFailedEvent;
 use Codeception\Event\SuiteEvent;
 use Codeception\Event\StepEvent;
 
@@ -117,4 +118,18 @@ public function stepBefore(StepEvent $stepEvent)
         $this->emptyStep = false;
         $this->getLifecycle()->fire(new StepStartedEvent($stepName));
     }
+
+    /**
+     * Override of parent method, fires StepFailedEvent if step has failed (for xml output)
+     * @param StepEvent $stepEvent
+     * @throws \Yandex\Allure\Adapter\AllureException
+     * @return void
+     */
+    public function stepAfter(StepEvent $stepEvent = null)
+    {
+        if ($stepEvent->getStep()->hasFailed()) {
+            $this->getLifecycle()->fire(new StepFailedEvent());
+        }
+        $this->getLifecycle()->fire(new StepFinishedEvent());
+    }
 }

src/Magento/FunctionalTestingFramework/Console/RunTestFailedCommand.php

@@ -135,6 +135,11 @@ private function getFailedTestList()
                     if ($suiteName == self::DEFAULT_TEST_GROUP) {
                         array_push($failedTestDetails['tests'], $testName);
                     } else {
+                        // Trim potential suite_parallel_0 to suite_parallel
+                        $suiteNameArray = explode("_", $suiteName);
+                        if (is_numeric(array_pop($suiteNameArray))) {
+                            $suiteName = implode("_", $suiteNameArray);
+                        }
                         $failedTestDetails['suites'] = array_merge_recursive(
                             $failedTestDetails['suites'],
                             [$suiteName => [$testName]]