From 8cdf9bc5e51e80fa2a8794051a763b00c417555d Mon Sep 17 00:00:00 2001 From: jrfnl Date: Wed, 10 May 2023 21:28:26 +0200 Subject: [PATCH] CI/QA: start recording code coverage This commit makes the necessary changes to start recording and uploading code coverage data to Coveralls. Includes: * Updating the GH Actions `test` workflow to run the high/low PHP version builds with code coverage and upload the results. * Updating the PHPUnit config to record coverage and show an inline code coverage summary. * Ignoring the potentially generated code coverage files in the `build/*` directory to prevent it being committed. * Adding scripts to the `composer.json` file to run code coverage. * Adding a code coverage badge to the README. Notes: * Coverage will be recorded using `Xdebug` as the code coverage driver. * Recording code coverage can be slow and it is not needed to run it on each build to still get a good idea of the code coverage. With that in mind, code coverage is only recorded for high/low PHP. * The `php-coveralls/php-coveralls` package is used to convert the coverage information from a format as generated by PHPUnit to a format as can be consumed by Coveralls. - As this package is only needed for the upload to Coveralls, the package has **not** been added it to the `composer.json` file, but will be (global) installed in the GH Actions workflow only. - The php-coveralls/php-coveralls package is not 100% compatible with PHP 8.x (yet), so for uploading the coverage generated using PHP 8.x, we normally switch to PHP 7.4 for uploading the code coverage reports, but you can only switch PHP version once per job and as we already do this for building the container, we cannot do it again. The chance of hitting the PHP 8.x problem is small anyway, so we'll just have to risk it for now. * Coveralls requires a token to identify the repo and prevent unauthorized uploads. This token has been added to the repository secrets. People with admin access to the GH repo automatically also have access to the admin settings in Coveralls. If ever needed, the Coveralls token can be found (and regenerated) in the Coveralls admin for a repo. After regeneration, the token as stored in the GH repo Settings -> Secrets and Variables -> Actions -> Repository secrets should be updated. * As the workflow sends multiple code coverage reports to Coveralls, we need to tell it to process those reports in parallel and give each report a unique name. That's what the `COVERALLS_PARALLEL` and `COVERALLS_FLAG_NAME` settings are about. This is also why the Clover files for the unit and the integration tests in the PHPUnit `logging` section have different names. Additionally, it is why the name for the Clover file for the multi-site run in the integration tests is overruled in the workflow. This allows us to save the coverage results for the single site and the multi site test runs in separate files and to upload both files to Coveralls for a more complete code coverage picture. * The `coveralls-finish` job will signal to Coveralls that all reports have been uploaded. This basically gives the "okay" to Coveralls to report on the changes in code coverage via a comment on a GitHub PR as well as via a status check. The pertinent Coveralls settings used for this repo are: * "Only send aggregate Coverage updates to SCM": enabled * "Leave comments": enabled * "(Comment) Format": detailed * "Use Status API": enabled * "Coverage threshold for failure": 36% * "Coverage decrease threshold for failure": 1.0% --- .github/workflows/test.yml | 83 ++++++++++++++++++++++++++++++++++-- .gitignore | 1 + README.md | 1 + composer.json | 6 +++ phpunit-integration.xml.dist | 6 +++ phpunit.xml.dist | 6 +++ 6 files changed, 99 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0a74ffd1e73..885f6b4e136 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,15 @@ jobs: strategy: matrix: - php_version: ["7.2", "7.4", "8.0", "8.1", "8.2"] + php_version: ["7.4", "8.0", "8.1"] + coverage: [false] + + # Run code coverage only on high/low PHP. + include: + - php_version: 7.2 + coverage: true + - php_version: 8.2 + coverage: true name: "Unit Test: PHP ${{ matrix.php_version }}" @@ -59,7 +67,7 @@ jobs: with: php-version: ${{ matrix.php_version }} ini-values: zend.assertions=1, error_reporting=-1, display_errors=On - coverage: none + coverage: ${{ matrix.coverage == true && 'xdebug' || 'none' }} # The PHP platform requirement would prevent updating the test utilities to the appropriate versions. # As long as the `composer update` is run selectively to only update the test utils, removing this is fine. @@ -81,8 +89,26 @@ jobs: custom-cache-suffix: $(/bin/date -u --date='last Mon' "+%F") - name: Run unit tests + if: ${{ matrix.coverage == false }} run: composer test + - name: Run the unit tests with code coverage + if: ${{ matrix.coverage == true }} + run: composer coverage + + # Global install is used to prevent a conflict with the local composer.lock in PHP 8.0+. + - name: Install Coveralls + if: ${{ success() && matrix.coverage == true }} + run: composer global require php-coveralls/php-coveralls:"^2.5.3" --no-interaction + + - name: Upload coverage results to Coveralls + if: ${{ success() && matrix.coverage == true }} + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: unit-php-${{ matrix.php_version }} + run: php-coveralls -v -x build/logs/clover.xml + integration-test: runs-on: ubuntu-latest @@ -92,29 +118,35 @@ jobs: - php_version: "7.2" wp_version: "6.1" multisite: true + coverage: true - php_version: "7.3" wp_version: "trunk" multisite: true + coverage: false - php_version: "7.4" wp_version: "latest" multisite: false + coverage: false # WP 5.6 is the earliest version which (sort of) supports PHP 8.0. - php_version: "8.0" wp_version: "6.1" multisite: false + coverage: false # WP 5.9 is the earliest version which (sort of) supports PHP 8.1. - php_version: "8.1" wp_version: "latest" multisite: true + coverage: false # WP 6.1 is the earliest version which supports PHP 8.2. - php_version: "8.2" wp_version: "6.2" multisite: true + coverage: true name: "Integration Test: PHP ${{ matrix.php_version }} | WP ${{ matrix.wp_version }}${{ matrix.multisite == true && ' (+ ms)' || '' }}" @@ -162,7 +194,7 @@ jobs: with: php-version: ${{ matrix.php_version }} ini-values: zend.assertions=1, error_reporting=-1, display_errors=On - coverage: none + coverage: ${{ matrix.coverage == true && 'xdebug' || 'none' }} # The PHP platform requirement would prevent updating the test utilities to the appropriate versions. # As long as the `composer update` is run selectively to only update the test utils, removing this is fine. @@ -184,10 +216,53 @@ jobs: run: config/scripts/install-wp-tests.sh wordpress_test root '' 127.0.0.1:3306 ${{ matrix.wp_version }} - name: Run unit tests - single site + if: ${{ matrix.coverage == false }} run: composer integration-test - name: Run unit tests - multisite - if: ${{ matrix.multisite == true }} + if: ${{ matrix.multisite == true && matrix.coverage == false }} run: composer integration-test env: WP_MULTISITE: 1 + + - name: Run unit tests with code coverage - single site + if: ${{ matrix.coverage == true }} + run: composer integration-coverage + + - name: Run unit tests with code coverage - multisite + if: ${{ matrix.multisite == true && matrix.coverage == true }} + run: composer integration-coverage -- --coverage-clover build/logs/clover-integration-ms.xml + env: + WP_MULTISITE: 1 + + # Global install is used to prevent a conflict with the local composer.lock in PHP 8.0+. + - name: Install Coveralls + if: ${{ success() && matrix.coverage == true }} + run: composer global require php-coveralls/php-coveralls:"^2.5.3" --no-interaction + + - name: Upload coverage results to Coveralls - single site + if: ${{ success() && matrix.coverage == true }} + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: intgr-php-${{ matrix.php_version }}-wp-${{ matrix.wp_version }} + run: php-coveralls -v -x build/logs/clover-integration.xml + + - name: Upload coverage results to Coveralls - multisite + if: ${{ success() && matrix.multisite == true && matrix.coverage == true }} + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_TOKEN }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: intgr-php-${{ matrix.php_version }}-wp-${{ matrix.wp_version }}-ms + run: php-coveralls -v -x build/logs/clover-integration-ms.xml + + coveralls-finish: + needs: [unit-test, integration-test] + runs-on: ubuntu-latest + + steps: + - name: Coveralls Finished + uses: coverallsapp/github-action@v2 + with: + github-token: ${{ secrets.COVERALLS_TOKEN }} + parallel-finished: true diff --git a/.gitignore b/.gitignore index accdaf23c87..bd2384ee3eb 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ phpcs.xml phpunit.xml phpunit-integration.xml .phpunit.result.cache +build/ ############ ## OSes diff --git a/README.md b/README.md index ef2863cd0e9..33324e46ddb 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ [![TestJS](https://github.com/Yoast/wordpress-seo/actions/workflows/jstest.yml/badge.svg)](https://github.com/Yoast/wordpress-seo/actions/workflows/jstest.yml) [![Test](https://github.com/Yoast/wordpress-seo/actions/workflows/test.yml/badge.svg)](https://github.com/Yoast/wordpress-seo/actions/workflows/test.yml) [![Deployment](https://github.com/Yoast/wordpress-seo/actions/workflows/deploy.yml/badge.svg)](https://github.com/Yoast/wordpress-seo/actions/workflows/deploy.yml) +[![Coverage Status](https://coveralls.io/repos/github/Yoast/wordpress-seo/badge.svg?branch=trunk)](https://coveralls.io/github/Yoast/wordpress-seo?branch=trunk) [![Stable Version](https://poser.pugx.org/yoast/wordpress-seo/v/stable.svg)](https://packagist.org/packages/yoast/wordpress-seo) [![License](https://poser.pugx.org/yoast/wordpress-seo/license.svg)](https://packagist.org/packages/yoast/wordpress-seo) diff --git a/composer.json b/composer.json index 6eeacfda593..a7c7470d13f 100644 --- a/composer.json +++ b/composer.json @@ -56,9 +56,15 @@ }, "scripts": { "test": [ + "@php ./vendor/phpunit/phpunit/phpunit --no-coverage" + ], + "coverage": [ "@php ./vendor/phpunit/phpunit/phpunit" ], "integration-test": [ + "@php ./vendor/phpunit/phpunit/phpunit -c phpunit-integration.xml.dist --no-coverage" + ], + "integration-coverage": [ "@php ./vendor/phpunit/phpunit/phpunit -c phpunit-integration.xml.dist" ], "lint": [ diff --git a/phpunit-integration.xml.dist b/phpunit-integration.xml.dist index a8d40164a53..702177976bf 100644 --- a/phpunit-integration.xml.dist +++ b/phpunit-integration.xml.dist @@ -37,4 +37,10 @@ + + + + + + diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 4ef46c98db9..a5def43698f 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -38,4 +38,10 @@ + + + + + +