Feature/reporter ext and splitter for failed tests (#66)

* Create php.yml

* Added a FailedTestsReporter which catched all failed Tests and write it to the failedTests.txt

* outsourced method which groups the files/tests

* FailedTestSplitterTask.php created, Tests for the Task

* Updated Readme.md, fixed Returntype in trait

README.md

@@ -79,6 +79,25 @@ $this->taskSplitTestsByTime(5)
 
 this command need run all tests with `Codeception\Task\TimeReporter` for collect execution time. If you want just split tests between group (and not execute its) you can use SplitTestsByGroups. **Please be aware**: This task will not consider any 'depends' annotation!
 
+### SplitFailedTests
+
+Enable extension for collect failed tests if you use taskSplitFailedTests
+
+```
+extensions:
+    enabled:
+        - Codeception\Task\Extension\FailedTestsReporter
+```
+
+Load the failed Tests from a reportfile into the groups:
+- Default report path is: `Configuration::outputDir() . 'failedTests.txt'`
+```php
+$this
+    ->taskSplitFailedTests(5)
+    ->setReportPath('tests/_output/' . FailedTestsReporter::REPORT_NAME)
+    ->groupsTo('tests/_data/group_')
+    ->run();
+```
 ### MergeXmlReports
 
 Mergex several XML reports:

src/Extension/FailedTestsReporter.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codeception\Task\Extension;
+
+use Codeception\Event\FailEvent;
+use Codeception\Event\TestEvent;
+use Codeception\Events;
+use Codeception\Extension;
+use Codeception\Test\Descriptor;
+
+/**
+ * Class FailedTestsReporter - reports the failed tests to a reportfile
+ * Modify the codeception.yml to enable this extension:
+ *  extensions:
+ *      enabled:
+ *          - Codeception\Task\Extension\FailedTestsReporter
+ */
+class FailedTestsReporter extends Extension
+{
+    /** @var string */
+    public const REPORT_NAME = 'failedTests.txt';
+
+    /** @var string $reportFile */
+    private $reportFile = self::REPORT_NAME;
+
+    /** @var array $failedTests */
+    private $failedTests = [];
+
+    /**
+     * @var string[] $events
+     */
+    public static $events = [
+        Events::TEST_FAIL => 'afterFail',
+        Events::RESULT_PRINT_AFTER => 'endRun',
+    ];
+
+    /**
+     * Event after each failed test - collect the failed test
+     * @param FailEvent $event
+     */
+    public function afterFail(FailEvent $event): void
+    {
+        $this->failedTests[] = $this->getTestname($event);
+    }
+
+    /**
+     * Event after all Tests - write failed tests to reportfile
+     */
+    public function endRun(): void
+    {
+        if (empty($this->failedTests)) {
+            return;
+        }
+
+        $file = $this->getLogDir() . $this->reportFile;
+        if (is_file($file)) {
+            unlink($file); // remove old reportFile
+        }
+
+        file_put_contents($file, implode(PHP_EOL, $this->failedTests));
+    }
+
+    /**
+     * @param TestEvent $e
+     * @return false|string
+     */
+    public function getTestname(TestEvent $e): string
+    {
+        $name = Descriptor::getTestFullName($e->getTest());
+
+        return substr(str_replace($this->getRootDir(), '', $name), 1);
+    }
+}

src/Splitter/FailedTestSplitterTask.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Codeception\Task\Splitter;
+
+use Codeception\Configuration;
+use Codeception\Task\Extension\FailedTestsReporter;
+use RuntimeException;
+
+class FailedTestSplitterTask extends TestsSplitter
+{
+    /** @var string */
+    private $reportPath = null;
+
+    /**
+     * @return string
+     * @throws \Codeception\Exception\ConfigurationException
+     */
+    public function getReportPath(): string
+    {
+        return $this->reportPath ?? (Configuration::logDir() . FailedTestsReporter::REPORT_NAME);
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function run()
+    {
+        $this->claimCodeceptionLoaded();
+        $reportPath = $this->getReportPath();
+
+        if (!@file_exists($reportPath)) {
+            throw new RuntimeException(
+                'The reportfile "failedTests.txt" did not exists.'
+            );
+        }
+
+        $this->splitToGroupFiles(
+            $this->filter(
+                explode(
+                    PHP_EOL,
+                    file_get_contents($reportPath)
+                )
+            )
+        );
+    }
+
+    /**
+     * @param string $reportPath
+     * @return FailedTestSplitterTask
+     */
+    public function setReportPath(string $reportPath): FailedTestSplitterTask
+    {
+        if (empty($reportPath)) {
+            throw new \InvalidArgumentException('The reportPath could not be empty!');
+        }
+
+        $this->reportPath = $reportPath;
+
+        return $this;
+    }
+}

src/Splitter/TestFileSplitterTask.php

@@ -38,25 +38,14 @@ public function run()
             ->in($this->projectRoot ?: getcwd())
             ->exclude($this->excludePath);
 
-        $i = 0;
-        $groups = [];
-
-        $this->printTaskInfo('Processing ' . count($files) . ' files');
-        $files = $this->filter(iterator_to_array($files->getIterator()));
-
-        // splitting tests by groups
-        /** @var SplFileInfo $file */
-        foreach ($files as $file) {
-            $groups[($i % $this->numGroups) + 1][] = $file->getRelativePathname();
-            $i++;
-        }
-
-        // saving group files
-        foreach ($groups as $i => $tests) {
-            $filename = $this->saveTo . $i;
-            $this->printTaskInfo("Writing $filename");
-            file_put_contents($filename, implode("\n", $tests));
-        }
+        $this->splitToGroupFiles(
+            array_map(
+                static function (SplFileInfo $fileInfo): string {
+                    return $fileInfo->getRelativePathname();
+                },
+                $this->filter(iterator_to_array($files->getIterator()))
+            )
+        );
     }
 
     /**

src/Splitter/TestsSplitter.php

@@ -213,4 +213,31 @@ protected function doCodeceptLoaderExists(): bool
     {
         return class_exists('\Codeception\Test\Loader');
     }
+
+    /**
+     * Splitting array of files to the group files
+     * @param string[] $files - the relative path of the Testfile with or without test function
+     * @example $this->splitToGroupFiles(['tests/FooCest.php', 'tests/BarTest.php:testBarReturn']);
+     */
+    protected function splitToGroupFiles(array $files): void
+    {
+        $i = 0;
+        $groups = [];
+
+        $this->printTaskInfo('Processing ' . count($files) . ' files');
+
+        // splitting tests by groups
+        /** @var string $file */
+        foreach ($files as $file) {
+            $groups[($i % $this->numGroups) + 1][] = $file;
+            $i++;
+        }
+
+        // saving group files
+        foreach ($groups as $i => $tests) {
+            $filename = $this->saveTo . $i;
+            $this->printTaskInfo("Writing $filename");
+            file_put_contents($filename, implode("\n", $tests));
+        }
+    }
 }

src/Splitter/TestsSplitterTrait.php

@@ -4,35 +4,47 @@
 
 namespace Codeception\Task\Splitter;
 
+use Robo\Collection\CollectionBuilder;
+
 trait TestsSplitterTrait
 {
     /**
      * @param int $numGroups
      *
-     * @return TestsSplitterTask
+     * @return TestsSplitterTask|CollectionBuilder
      */
-    protected function taskSplitTestsByGroups(int $numGroups): TestsSplitterTask
+    protected function taskSplitTestsByGroups(int $numGroups)
     {
         return $this->task(TestsSplitterTask::class, $numGroups);
     }
 
     /**
      * @param int $numGroups
      *
-     * @return TestFileSplitterTask
+     * @return TestFileSplitterTask|CollectionBuilder
      */
-    protected function taskSplitTestFilesByGroups(int $numGroups): TestFileSplitterTask
+    protected function taskSplitTestFilesByGroups(int $numGroups)
     {
         return $this->task(TestFileSplitterTask::class, $numGroups);
     }
 
     /**
-     * @param $numGroups
+     * @param int $numGroups
      *
-     * @return TestFileSplitterTask
+     * @return TestFileSplitterTask|CollectionBuilder
      */
-    protected function taskSplitTestsByTime($numGroups): TestFileSplitterTask
+    protected function taskSplitTestsByTime(int $numGroups)
     {
         return $this->task(SplitTestsByTimeTask::class, $numGroups);
     }
+
+    /**
+     * @param int $numGroups
+     *
+     * @return TestFileSplitterTask|CollectionBuilder
+     */
+    protected function taskSplitFailedTests(int $numGroups)
+    {
+        return $this->task(FailedTestSplitterTask::class, $numGroups);
+    }
 }

tests/Extension/FailedTestsReporterTest.php

@@ -0,0 +1,81 @@
+<?php
+
+namespace Tests\Codeception\Task\Extension;
+
+use Codeception\Event\FailEvent;
+use Codeception\Event\TestEvent;
+use Codeception\Task\Extension\FailedTestsReporter;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * Class FailedTestsReporterTest
+ * @coversDefaultClass \Codeception\Task\Extension\FailedTestsReporter
+ */
+class FailedTestsReporterTest extends TestCase
+{
+    private $failedTests = [
+        ['testname' => 'tests/acceptance/bar/baz.php:testA',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testB',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testC',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testD',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testE',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testF',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testG',],
+        ['testname' => 'tests/acceptance/bar/baz.php:testH',],
+    ];
+
+    /**
+     * @covers ::endRun
+     */
+    public function testEndRun(): void
+    {
+        $reporter = $this->getMockBuilder(FailedTestsReporter::class)
+            ->disableOriginalConstructor()
+            ->onlyMethods(['getTestname', 'getLogDir'])
+            ->getMock();
+
+        $reporter->method('getLogDir')->willReturn(TEST_PATH . '/result/');
+
+        // prepare Mocks for Test
+        $testEvents = [];
+        foreach ($this->failedTests as $test) {
+            $eventMock = $this->getMockBuilder(FailEvent::class)
+                ->disableOriginalConstructor()
+                ->getMock();
+
+            $testEvents[] = [
+                'mock' => $eventMock,
+                'testname' => $test['testname']
+            ];
+        }
+
+        // get Testname by the TestEventMock
+        $reporter
+            ->method('getTestname')
+            ->withConsecutive(
+                ...array_map(
+                    static function (FailEvent $event): array {
+                        return [$event];
+                    },
+                    array_column($testEvents, 'mock')
+                )
+            )
+            ->willReturnOnConsecutiveCalls(...array_column($testEvents, 'testname'));
+
+        foreach ($testEvents as $event) {
+            $reporter->afterFail($event['mock']);
+        }
+
+        $reporter->endRun();
+        $file = TEST_PATH . '/result/failedTests.txt';
+        $this->assertFileExists($file);
+        $content = explode(PHP_EOL, file_get_contents($file));
+        $this->assertCount(8, $content);
+    }
+
+    protected function tearDown(): void
+    {
+        parent::tearDown(); // TODO: Change the autogenerated stub
+        unlink(TEST_PATH . '/result/failedTests.txt');
+    }
+}