From 4294d322f695632d4b2b9a2122c8a6ef130ae948 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 17 Jan 2023 14:50:00 +0100 Subject: [PATCH 01/14] GH-32: 2023 Q1 dependency updates. --- package.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/package.json b/package.json index 1a90465..2f964c5 100644 --- a/package.json +++ b/package.json @@ -25,23 +25,23 @@ }, "homepage": "https://github.com/fluid-project/fluid-testem/", "dependencies": { - "glob": "7.1.6", - "fluid-express": "1.0.16", - "infusion": "3.0.0-dev.20201113T153152Z.32176dcbe.FLUID-6145", - "istanbul-lib-instrument": "4.0.3", - "mkdirp": "1.0.4", - "node-jqunit": "1.1.8", + "glob": "8.1.0", + "fluid-express": "1.0.18", + "infusion": "4.6.0", + "istanbul-lib-instrument": "5.2.1", + "mkdirp": "2.1.1", + "node-jqunit": "1.1.9", "nyc": "15.1.0", - "rimraf": "3.0.2", - "testem": "3.2.0" + "rimraf": "4.1.0", + "testem": "3.10.1" }, "devDependencies": { - "eslint": "7.17.0", - "eslint-config-fluid": "2.0.0", - "form-data": "3.0.0", - "fluid-lint-all": "1.0.0", + "eslint": "8.32.0", + "eslint-config-fluid": "2.1.1", + "form-data": "4.0.0", + "fluid-lint-all": "1.2.6", "fluid-webdriver": "1.1.2", - "kettle": "2.0.0", + "kettle": "2.3.0", "qunitjs": "2.4.1" } } From 04ab06579fa52a85efbbfb8501e33fe1b3b600d3 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 17 Jan 2023 14:55:08 +0100 Subject: [PATCH 02/14] GH-32: Fixed linting errors resulting from newer lint checks. --- src/js/lib/pathUtils.js | 2 ++ src/js/lib/resolveSafely.js | 2 ++ src/js/testem-component.js | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/js/lib/pathUtils.js b/src/js/lib/pathUtils.js index 59c5bd3..d3d0fbb 100644 --- a/src/js/lib/pathUtils.js +++ b/src/js/lib/pathUtils.js @@ -20,6 +20,7 @@ fluid.testem.extractLastPathSegment = function (rawPath, leader) { return fluid.testem.forceLeadingSlash(path.basename(resolvedPath), leader); }; +/* eslint-disable jsdoc/require-returns-check */ /** * * Ensure that the path segment is preceded by a leader (defaults to a leading slash). @@ -29,6 +30,7 @@ fluid.testem.extractLastPathSegment = function (rawPath, leader) { * @return {String} - The path, updated as needed to ensure that it begins with `leader`. * */ +/* eslint-enable jsdoc/require-returns-check */ fluid.testem.forceLeadingSlash = function (rawPath, leader) { leader = (leader || leader === "") ? leader : "/"; if (typeof rawPath === "string") { diff --git a/src/js/lib/resolveSafely.js b/src/js/lib/resolveSafely.js index 6f47bf6..7cda182 100644 --- a/src/js/lib/resolveSafely.js +++ b/src/js/lib/resolveSafely.js @@ -18,6 +18,7 @@ fluid.testem.resolvePathSafely = function (basePath, subPath) { } }; +/* eslint-disable jsdoc/require-returns-check */ /** * * Another wrapper to ensure that invalid or missing paths do not break the overall lifecycle of a testem component. @@ -25,6 +26,7 @@ fluid.testem.resolvePathSafely = function (basePath, subPath) { * @param {String} path - The path to resolve. * @return {String} - The resolved path. */ +/* eslint-enable jsdoc/require-returns-check */ fluid.testem.resolveFluidModulePathSafely = function (path) { try { var resolvedPath = fluid.module.resolvePath(path); diff --git a/src/js/testem-component.js b/src/js/testem-component.js index e523d9d..0dcf202 100644 --- a/src/js/testem-component.js +++ b/src/js/testem-component.js @@ -264,6 +264,7 @@ fluid.testem.cleanup = function (stage, cleanupDefs, rimrafOptions) { return togo; }; +/* eslint-disable jsdoc/require-returns-check */ /** * * Generate a unique subdirectory path based on a supplied prefix and suffix. @@ -273,6 +274,7 @@ fluid.testem.cleanup = function (stage, cleanupDefs, rimrafOptions) { * @param {String} suffix - A "suffix" that will be appended to the end of the filename. * @return {String} - The full path to the unique subdirectory. */ +/* eslint-enable jsdoc/require-returns-check */ fluid.testem.generateUniqueDirName = function (basePath, prefix, suffix) { try { var resolvedBasePath = fluid.testem.resolveFluidModulePathSafely(basePath); From 834ae23f4bc87297503a6a03fffc83f70fa3ea1c Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 17 Jan 2023 14:55:28 +0100 Subject: [PATCH 03/14] GH-32: Updated tested node versions and moved linting checks to the end. --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fff2a33..fdf0e2f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - node-version: [12.x, 14.x] + node-version: [16.x, 18.x] env: HEADLESS: true @@ -29,11 +29,11 @@ jobs: - name: Install Node.js dependencies run: npm install - - name: Lint Code - run: npm run lint - - name: Node Tests run: xvfb-run --auto-servernum npm test - name: Cleanup after xvfb uses: bcomnes/cleanup-xvfb@v1 + + - name: Lint Code + run: npm run lint From f295346a0122a5219b4af284925cbb95cc9626b1 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Wed, 18 Jan 2023 08:21:06 +0100 Subject: [PATCH 04/14] GH-32: Reverted rimraf update to fix stalling tests. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 2f964c5..87934e1 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,10 @@ "fluid-express": "1.0.18", "infusion": "4.6.0", "istanbul-lib-instrument": "5.2.1", - "mkdirp": "2.1.1", + "mkdirp": "2.1.3", "node-jqunit": "1.1.9", "nyc": "15.1.0", - "rimraf": "4.1.0", + "rimraf": "3.0.2", "testem": "3.10.1" }, "devDependencies": { From f6f27623ad29d80a273af3f73fa15f3f7a6eb702 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Wed, 18 Jan 2023 09:26:52 +0100 Subject: [PATCH 05/14] GH-32: Updated rimraf calls to use promises. --- package.json | 2 +- src/js/testem-component.js | 27 +++++++------------ tests/js/testem-component-functional-tests.js | 5 ++-- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/package.json b/package.json index 87934e1..af0a0e5 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "mkdirp": "2.1.3", "node-jqunit": "1.1.9", "nyc": "15.1.0", - "rimraf": "3.0.2", + "rimraf": "4.1.1", "testem": "3.10.1" }, "devDependencies": { diff --git a/src/js/testem-component.js b/src/js/testem-component.js index 0dcf202..d276873 100644 --- a/src/js/testem-component.js +++ b/src/js/testem-component.js @@ -125,16 +125,10 @@ fluid.testem.getTestemOptions = function (that) { fluid.testem.generateRimrafWrapper = function (path, rimrafOptions) { return function () { - var rimrafPromise = fluid.promise(); - rimraf(path, fluid.copy(rimrafOptions), function (rimrafError) { - if (rimrafError) { - rimrafPromise.reject(rimrafError); - } - else { - rimrafPromise.resolve(); - } - }); - return rimrafPromise; + var fluidPromise = fluid.promise(); + var rimrafPromise = rimraf(path, fluid.copy(rimrafOptions)); + rimrafPromise.then(fluidPromise.resolve, fluidPromise.reject); + return fluidPromise; }; }; @@ -207,13 +201,12 @@ fluid.testem.cleanupDir = function (cleanupDef, rimrafOptions) { promise.resolve(); } else { - rimraf(resolvedPath, fluid.copy(rimrafOptions), function (error) { - if (error) { - fluid.log(fluid.logLevel.WARN, "Error removing ", cleanupDef.name, " content:", error); - } - else { - fluid.log("Removed ", cleanupDef.name, " content..."); - } + var rimrafPromise = rimraf(resolvedPath, fluid.copy(rimrafOptions)); + rimrafPromise.then(function () { + fluid.log("Removed ", cleanupDef.name, " content..."); + promise.resolve(); + }, function (error) { + fluid.log(fluid.logLevel.WARN, "Error removing ", cleanupDef.name, " content:", error); promise.resolve(); }); } diff --git a/tests/js/testem-component-functional-tests.js b/tests/js/testem-component-functional-tests.js index a0a3032..eb37719 100644 --- a/tests/js/testem-component-functional-tests.js +++ b/tests/js/testem-component-functional-tests.js @@ -73,9 +73,8 @@ fluid.tests.testem.runner.runSingleTest = function (that, testDef) { var promise = fluid.promise(); var resolvedPathToRemove = fluid.module.resolvePath(dirToRemove); fluid.log("Removing dir '", resolvedPathToRemove, "'..."); - rimraf(resolvedPathToRemove, function (error) { - error ? promise.reject(error) : promise.resolve(); - }); + var rimRafPromise = rimraf(resolvedPathToRemove); + rimRafPromise.then(promise.resolve, promise.reject); return promise; }); } From 444e3b41e5ebc06d0091586df7dab4e5e79b1ce1 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Wed, 18 Jan 2023 16:40:06 +0100 Subject: [PATCH 06/14] GH-32: Updated to newer fluid-express. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index af0a0e5..05ea667 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "homepage": "https://github.com/fluid-project/fluid-testem/", "dependencies": { "glob": "8.1.0", - "fluid-express": "1.0.18", + "fluid-express": "1.0.19", "infusion": "4.6.0", "istanbul-lib-instrument": "5.2.1", "mkdirp": "2.1.3", From 1b95a39482b53035d4e7224e9f54d9b90d4b2a50 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Thu, 19 Jan 2023 09:52:23 +0100 Subject: [PATCH 07/14] GH-32: Removed wrapping fluid promise from several rimraf calls. --- src/js/testem-component.js | 5 +---- tests/js/testem-component-functional-tests.js | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/js/testem-component.js b/src/js/testem-component.js index d276873..724b6be 100644 --- a/src/js/testem-component.js +++ b/src/js/testem-component.js @@ -125,10 +125,7 @@ fluid.testem.getTestemOptions = function (that) { fluid.testem.generateRimrafWrapper = function (path, rimrafOptions) { return function () { - var fluidPromise = fluid.promise(); - var rimrafPromise = rimraf(path, fluid.copy(rimrafOptions)); - rimrafPromise.then(fluidPromise.resolve, fluidPromise.reject); - return fluidPromise; + return rimraf(path, fluid.copy(rimrafOptions)); }; }; diff --git a/tests/js/testem-component-functional-tests.js b/tests/js/testem-component-functional-tests.js index eb37719..a5d5ce1 100644 --- a/tests/js/testem-component-functional-tests.js +++ b/tests/js/testem-component-functional-tests.js @@ -70,12 +70,9 @@ fluid.tests.testem.runner.runSingleTest = function (that, testDef) { // Needed to avoid problems with "failure" tests. if (dirToRemove) { cleanupPromises.push(function () { - var promise = fluid.promise(); var resolvedPathToRemove = fluid.module.resolvePath(dirToRemove); fluid.log("Removing dir '", resolvedPathToRemove, "'..."); - var rimRafPromise = rimraf(resolvedPathToRemove); - rimRafPromise.then(promise.resolve, promise.reject); - return promise; + return rimraf(resolvedPathToRemove); }); } }); From 052ff7acb0d4d45ab877c40f0663403b89002d9a Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 21 Feb 2023 09:53:37 +0100 Subject: [PATCH 08/14] GH-32: Nudged dependencies that have updated since the pull was started. --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 05ea667..886b6c5 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,11 @@ "mkdirp": "2.1.3", "node-jqunit": "1.1.9", "nyc": "15.1.0", - "rimraf": "4.1.1", + "rimraf": "4.1.2", "testem": "3.10.1" }, "devDependencies": { - "eslint": "8.32.0", + "eslint": "8.34.0", "eslint-config-fluid": "2.1.1", "form-data": "4.0.0", "fluid-lint-all": "1.2.6", From 53e44c19518563549d5f1a9c48e31879172afe54 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 21 Feb 2023 09:58:36 +0100 Subject: [PATCH 09/14] NOGH: Updated workflow actions to use newer versions of plugins to avoid node@12 warnings. --- .github/workflows/main.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fdf0e2f..aec1bb9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,10 +19,10 @@ jobs: HEADLESS: true steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v3 with: node-version: ${{ matrix.node-version }} @@ -32,6 +32,13 @@ jobs: - name: Node Tests run: xvfb-run --auto-servernum npm test + - name: Archive Code Coverage Report + continue-on-error: true + uses: actions/upload-artifact@v3 + with: + name: Code Coverage Report + path: reports/ + - name: Cleanup after xvfb uses: bcomnes/cleanup-xvfb@v1 From 486a5d3970dc1e717734c30d5c704299ac76117d Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 21 Feb 2023 13:39:00 +0100 Subject: [PATCH 10/14] GH-32: Removed wildcards from pretest rimraf cleanup. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 886b6c5..117a9fa 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ }, "scripts": { "lint": "fluid-lint-all", - "pretest": "rimraf coverage/* reports/* instrumented/*", + "pretest": "rimraf coverage reports instrumented", "test": "nyc node tests/all-tests.js" }, "repository": { From 22237d09476079e0b70321b38aaeafae1f730cad Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Tue, 21 Feb 2023 19:28:42 +0100 Subject: [PATCH 11/14] GH-32: Excluded IE and Chrome variants from test runs. --- tests/harness.js | 2 +- tests/rollup-fixtures/testem.js | 2 +- tests/testem-fixtures/coverage-fixtures/testem-no-coverage.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/harness.js b/tests/harness.js index 759d267..18a755c 100644 --- a/tests/harness.js +++ b/tests/harness.js @@ -40,7 +40,7 @@ fluid.defaults("fluid.tests.testem.harness", { nm: "%fluid-testem/node_modules" }, testemOptions: { - skip: "Safari,PhantomJS" + skip: "Safari,PhantomJS,IE,Brave,Opera" }, listeners: { "onTestemStart.logTestemOptions": { diff --git a/tests/rollup-fixtures/testem.js b/tests/rollup-fixtures/testem.js index f97255b..2f3d127 100644 --- a/tests/rollup-fixtures/testem.js +++ b/tests/rollup-fixtures/testem.js @@ -18,7 +18,7 @@ var testemComponent = fluid.testem.instrumentation({ node_modules: "%fluid-testem/node_modules" }, testemOptions: { - skip: "Safari,PhantomJS" + skip: "Safari,PhantomJS,IE,Brave,Opera" }, coveragePort: 7017 }); diff --git a/tests/testem-fixtures/coverage-fixtures/testem-no-coverage.js b/tests/testem-fixtures/coverage-fixtures/testem-no-coverage.js index 285eef4..acfe2a7 100644 --- a/tests/testem-fixtures/coverage-fixtures/testem-no-coverage.js +++ b/tests/testem-fixtures/coverage-fixtures/testem-no-coverage.js @@ -25,7 +25,7 @@ var testemComponent = fluid.testem.base({ } }, testemOptions: { - skip: "Safari,PhantomJS" + skip: "Safari,PhantomJS,IE,Brave,Opera" } }); From e4307aa35d745923d99be6b99ac7c4e5ea4456be Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Mon, 13 Mar 2023 19:24:03 +0100 Subject: [PATCH 12/14] GH-35: Removed legacy "headless" support. --- src/js/testem-component.js | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/src/js/testem-component.js b/src/js/testem-component.js index 724b6be..0808ce2 100644 --- a/src/js/testem-component.js +++ b/src/js/testem-component.js @@ -310,11 +310,6 @@ fluid.testem.stopServer = function (that) { } }; -// An expander to allow us to toggle "HEADLESS" mode with an environment variable. -fluid.testem.constructBrowserArgs = function (browserArgs, headlessBrowserArgs) { - return (process.env.HEADLESS && headlessBrowserArgs) || browserArgs; -}; - /** * * Construct a full set of Testem proxy configuration options based on component options. @@ -415,25 +410,13 @@ fluid.defaults("fluid.testem.base", { "--disable-new-zip-unpacker" ] }, - "headlessBrowserArgs": { - "Firefox": [ - "--no-remote", - "--headless" - ], - // See this ticket for details on the minimum options required to get "headless" Chrome working: https://github.com/testem/testem/issues/1106#issuecomment-298841383 - "Chrome": [ - "--disable-gpu", - "--headless", - "--remote-debugging-port=9222" - ] - }, testemOptions: { // The timeout options and Chrome browser args are workaround to minimize "browser disconnect" errors. // https://github.com/testem/testem/issues/777 browser_disconnect_timeout: 300, // Five minutes browser_start_timeout: 300, timeout: 300, - browser_args: "@expand:fluid.testem.constructBrowserArgs({that}.options.browserArgs, {that}.options.headlessBrowserArgs)", + browser_args: "{that}.options.browserArgs", framework: "qunit", tap_quiet_logs: true, report_file: { From ed537169431652f0dd00a772c85f08de6d61d927 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Mon, 13 Mar 2023 19:24:53 +0100 Subject: [PATCH 13/14] GH-36: GH-34: Add support for environment variables to control the behaviour of the `fluid.testem` grade in individual runs. --- README.md | 16 ++-- docs/testem-component.md | 79 ++++++++++++++--- src/js/testem-component.js | 87 +++++++++++++++++-- tests/js/testem-component-functional-tests.js | 31 +++---- .../coverage-fixtures/complete.html | 4 +- .../coverage-fixtures/incomplete.html | 2 +- 6 files changed, 176 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 406a781..d6b0fef 100644 --- a/README.md +++ b/README.md @@ -69,13 +69,13 @@ For more information, check out the documentation links below: You can run the tests using the command `npm test`. You are not required to have Testem installed globally to run the tests. -## A Warning about Internet Explorer 11 +## Running the tests in "Headless" Mode -There is currently [a bug in Testem](https://github.com/testem/testem/issues/1184) that results in IE11 windows opened -by Testem not being closed when the tests finish. Until that bug is resolved, if you are running tests from Windows, -you will need to close any open IE11 windows before launching the tests. +The default testem component provided by this package supports an environment variable that can be used to only run +tests in a subset of available browsers. (See [the component documentation](docs/testem-component.md) for more details). +Setting the `TESTEM_ENVIRONMENT` environment variable to `"headless"` will run tests using only headless Chrome and +Firefox. -## Running (Chrome) in "Headless" Mode - -If your system has [a new enough version of Chrome](https://developers.google.com/web/updates/2017/04/headless-chrome), -you can optionally run the tests in "headless" mode by setting the `HEADLESS` environment variable to a non-empty value. +Note: because of [a bug in Testem](https://github.com/testem/testem/issues/1377), tests that use Firefox require manual +input to complete on Windows. Setting the `TESTEM_ENVIRONMENT` environment variable to `"headless"` will allow tests to +run unattended on Windows. diff --git a/docs/testem-component.md b/docs/testem-component.md index 49b396c..a861695 100644 --- a/docs/testem-component.md +++ b/docs/testem-component.md @@ -1,18 +1,63 @@ # `fluid.testem` -The `fluid.testem` component is designed to: +The `fluid.testem` component is designed to launch browser-based tests in one or more browsers. By default, this +component: -1. Instrument the code under test. -2. Start test fixtures. -3. Run Testem tests. -4. Prepare a coverage report. -5. Clean up temporary files (instrumented source, browser data, coverage data). +1. Instruments the code under test. +2. Starts test fixtures. +3. Runs Testem tests. +4. Prepares a coverage report. +5. Cleans up temporary files (instrumented source, browser data, coverage data). -For basic usage instructions and requirements, see the [README file](../README.md). For detailed configuration options, -see below. +## Environment Variables + +The default grade `fluid.testem` supports a handful of environment variables that modify its behaviour. If you need +more control than these switches, you should consider extending one of the underlying grades instead (see below). + +### `DISABLE_COVERAGE` + +If this option is set, code will not be instrumented. This intended to support things like launching a configuration +file and debugging test problems locally, as in this example command on a system that supports a unix-like shell: + +```shell +DISABLE_COVERAGE=true node node_modules/testem/testem.js --file tests/testem.js +``` + +If you want to permanently disable instrumentation, you should look at the `fluid.testem.coverage` grade described +below. + +### `TESTEM_ENVIRONMENT` + +To help streamline local testing, two predefined "environments" are supported. These override the default testem +options to launch a minimal subset of browsers (only Firefox and Chrome). This is intended for cases in which you want +to be able to test for maximum compatibility by default, and to more quickly test incremental changes by only running +a subset of the available browsers. On a system that supports a unix-like shell, you might use this functionality with +commands like the following: + +```shell +TESTEM_ENVIRONMENT=headless npm test + +TESTEM_ENVIRONMENT=headed node node_modules/testem/testem.js --file tests/testem.js +``` + +The supported values are `"headed"` and `"headless"`, each of which is described below. + +#### "headed" + +The "minimal headed" environment launches only Firefox and Chrome. Note that if you do not have either of these +installed, runs launched with this environment will fail. Note also that because of +[a bug in testem](https://github.com/testem/testem/issues/1377), Firefox will not close itself when it is launched +"headed" on Windows. + +#### "headless" + +The "minimal headless" environment launches only Headless Firefox and Headless Chrome. Note that if you do not have +either of these installed, runs launched with this environment will fail. ## Component Options +These are the options supported by the default `fluid.testem` component when it is run with no environmental variables: + | Option | Type | Description | | ------------------------- | ----------- | ------------------------------------- | | `cwd` | `{String}` | Defaults to `process.cwd()`, i.e. the directory from which the script was called. | @@ -138,11 +183,9 @@ so that combined reports can be prepared. #### Additional Components -##### `fluid.testem.instrumentation` +##### `fluid.testem.base` -This grade instruments source itself and collects coverage data, but does not prepare a report at the end or remove the -coverage data during its cleanup phase. It is provided for use in combining reports from different test runs, such -as when running node tests and browser tests in the same package. See below for a detailed example. +The underlying base grade, without any instrumentation, coverage collection, or coverage reporting. ##### `fluid.testem.coverage` @@ -152,6 +195,18 @@ your instrumented code to use this grade. This grade only removes the temporary not any pre-instrumented code or coverage data. This grade is mainly provided for legacy setups, in almost all cases you should use the`fluid.testem.instrumentation` grade instead. +##### `fluid.testem.instrumentation` + +This grade instruments source itself and collects coverage data, but does not prepare a report at the end or remove the +coverage data during its cleanup phase. It is provided for use in combining reports from different test runs, such +as when running node tests and browser tests in the same package. See below for a detailed example. + +##### `fluid.testem.coverageReporting` + +This grade instruments code, collects coverage data, and generates a coverage report. If you are testing non-browser +code such as node scripts, if you save the coverage reporting data to the same directory, you can run this grade after +your node tests to produce a combined report. + #### Example: Combining Node Coverage with Browser Coverage. The key to preparing a combined report is to ensure that: diff --git a/src/js/testem-component.js b/src/js/testem-component.js index 0808ce2..6456983 100644 --- a/src/js/testem-component.js +++ b/src/js/testem-component.js @@ -38,8 +38,9 @@ fluid.testem.handleTestemLifecycleEvent = function (componentEvent, testemCallba fluid.log("Successfully reached the end of promise chain. Firing testem callback."); testemCallback(); }, - function () { - fluid.log("Promise chain terminated by promise rejection. Firing testem callback."); + function (error) { + fluid.log("Promise chain terminated by promise rejection:\n" + error); + fluid.log("Firing testem callback."); testemCallback(); } ); @@ -346,8 +347,44 @@ fluid.testem.constructProxies = function (sourceDirs, contentDirs, additionalPro return proxies; }; +fluid.testem.detectEnvironment = function (toMatch) { + var environmentValue = process.env.TESTEM_ENVIRONMENT; + if (!environmentValue) { return false; }; + + var toMatchRegexp = new RegExp(toMatch, "i"); + return environmentValue.match(toMatchRegexp); +}; + +fluid.testem.detectEnvironment.minimalHeadless = function () { + return fluid.testem.detectEnvironment("headless"); +}; + +fluid.testem.detectEnvironment.minimalHeaded = function () { + return fluid.testem.detectEnvironment("headed"); +}; + +fluid.contextAware.makeChecks({ + // predefined "environments" + "fluid.testem.detectEnvironment.minimalHeadless": "fluid.testem.detectEnvironment.minimalHeadless", + "fluid.testem.detectEnvironment.minimalHeaded": "fluid.testem.detectEnvironment.minimalHeaded" +}); + +fluid.defaults("fluid.testem.minimalHeadless", { + gradeNames: ["fluid.component"], + testemOptions: { + launch: "Headless Firefox,Headless Chrome" + } +}); + +fluid.defaults("fluid.testem.minimalHeaded", { + gradeNames: ["fluid.component"], + testemOptions: { + launch: "Firefox,Chrome" + } +}); + fluid.defaults("fluid.testem.base", { - gradeNames: ["fluid.component"], + gradeNames: ["fluid.component", "fluid.contextAware"], coveragePort: 7003, coverageUrl: { expander: { @@ -355,6 +392,22 @@ fluid.defaults("fluid.testem.base", { args: ["http://localhost:%port", { port: "{that}.options.coveragePort" }] } }, + + contextAwareness: { + environment: { + checks: { + minimalHeaded: { + contextValue: "{fluid.testem.detectEnvironment.minimalHeaded}", + gradeNames: ["fluid.testem.minimalHeaded"] + }, + minimalHeadless: { + contextValue: "{fluid.testem.detectEnvironment.minimalHeadless}", + gradeNames: ["fluid.testem.minimalHeadless"] + } + } + } + }, + cwd: process.cwd(), mergePolicy: { cleanup: "nomerge" @@ -631,8 +684,7 @@ fluid.defaults("fluid.testem.instrumentation", { } }); -// The default grade, which instruments source, collects coverage data, and generates reports. -fluid.defaults("fluid.testem", { +fluid.defaults("fluid.testem.coverageReporting", { gradeNames: ["fluid.testem.instrumentation"], reports: ["text-summary", "html", "json-summary"], cleanup: { @@ -657,3 +709,28 @@ fluid.defaults("fluid.testem", { } } }); + + +fluid.testem.enableCoverage = function () { + return process.env.DISABLE_COVERAGE ? false : true; +}; + +fluid.contextAware.makeChecks({ + // controls for instrumentation + coverage collection as well as coverage reporting. + "fluid.testem.enableCoverage": "fluid.testem.enableCoverage" +}); + +// The default grade, which supports an environment variable that disables instrumentation, coverage collection, and reporting. +fluid.defaults("fluid.testem", { + gradeNames: ["fluid.testem.base"], + contextAwareness: { + coverage: { + checks: { + enableCoverage: { + contextValue: "{fluid.testem.enableCoverage}", + gradeNames: ["fluid.testem.coverageReporting"] + } + } + } + } +}); diff --git a/tests/js/testem-component-functional-tests.js b/tests/js/testem-component-functional-tests.js index a5d5ce1..2cf4f97 100644 --- a/tests/js/testem-component-functional-tests.js +++ b/tests/js/testem-component-functional-tests.js @@ -150,21 +150,22 @@ fluid.defaults("fluid.tests.testem.runner", { } } }, - instrumentationTiming: { - name: "Confirm that long-running instrumentation does not interfere with coverage collection...", - configFile: "../testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js", - hasCoverage: true, - expectedCoverage: { - total: { - branches: { - total: 2, - covered: 2, - skipped: 0, - pct: 100 - } - } - } - }, + // TODO: Fix this + // instrumentationTiming: { + // name: "Confirm that long-running instrumentation does not interfere with coverage collection...", + // configFile: "../testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js", + // hasCoverage: true, + // expectedCoverage: { + // total: { + // branches: { + // total: 2, + // covered: 2, + // skipped: 0, + // pct: 100 + // } + // } + // } + // }, failure: { name: "Running a suite of tests with gross configuration errors...", configFile: "../testem-fixtures/failure-modes/testem-failure-modes.js", diff --git a/tests/testem-fixtures/coverage-fixtures/complete.html b/tests/testem-fixtures/coverage-fixtures/complete.html index f112c5e..b782394 100644 --- a/tests/testem-fixtures/coverage-fixtures/complete.html +++ b/tests/testem-fixtures/coverage-fixtures/complete.html @@ -1,6 +1,6 @@ - "Incomplete" coverage tests. + "Complete" coverage tests. @@ -28,7 +28,7 @@ -

"Incomplete Coverage" Tests

+

"Complete" Coverage Tests

diff --git a/tests/testem-fixtures/coverage-fixtures/incomplete.html b/tests/testem-fixtures/coverage-fixtures/incomplete.html index 76da141..99ec954 100644 --- a/tests/testem-fixtures/coverage-fixtures/incomplete.html +++ b/tests/testem-fixtures/coverage-fixtures/incomplete.html @@ -28,7 +28,7 @@ -

"Incomplete Coverage" Tests

+

"Incomplete" Coverage Tests

From 0061326bb34cd8814f0afc97b401e52b1619cb67 Mon Sep 17 00:00:00 2001 From: Tony Atkins Date: Mon, 13 Mar 2023 19:41:50 +0100 Subject: [PATCH 14/14] GH-36: Fixed 'slow instrumentation' tests broken by context awareness work. --- tests/js/testem-component-functional-tests.js | 31 +++++++++---------- .../testem-instrumentation-timing.js | 12 ++++++- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/tests/js/testem-component-functional-tests.js b/tests/js/testem-component-functional-tests.js index 2cf4f97..a5d5ce1 100644 --- a/tests/js/testem-component-functional-tests.js +++ b/tests/js/testem-component-functional-tests.js @@ -150,22 +150,21 @@ fluid.defaults("fluid.tests.testem.runner", { } } }, - // TODO: Fix this - // instrumentationTiming: { - // name: "Confirm that long-running instrumentation does not interfere with coverage collection...", - // configFile: "../testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js", - // hasCoverage: true, - // expectedCoverage: { - // total: { - // branches: { - // total: 2, - // covered: 2, - // skipped: 0, - // pct: 100 - // } - // } - // } - // }, + instrumentationTiming: { + name: "Confirm that long-running instrumentation does not interfere with coverage collection...", + configFile: "../testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js", + hasCoverage: true, + expectedCoverage: { + total: { + branches: { + total: 2, + covered: 2, + skipped: 0, + pct: 100 + } + } + } + }, failure: { name: "Running a suite of tests with gross configuration errors...", configFile: "../testem-fixtures/failure-modes/testem-failure-modes.js", diff --git a/tests/testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js b/tests/testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js index ee1d50e..447f1de 100644 --- a/tests/testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js +++ b/tests/testem-fixtures/coverage-fixtures/testem-instrumentation-timing.js @@ -41,7 +41,7 @@ fluid.tests.testem.instrumentationTiming.instrumentSlowly = function (that) { }; fluid.defaults("fluid.tests.testem.instrumentationTiming.harness", { - gradeNames: ["fluid.tests.testem.harness"], + gradeNames: ["fluid.tests.testem.harness", "fluid.testem.coverageReporting"], instrumentationDelay: 5000, testPages: ["complete.html"], cwd: __dirname, // required because we are working outside of our package root. @@ -51,6 +51,16 @@ fluid.defaults("fluid.tests.testem.instrumentationTiming.harness", { funcName: "fluid.tests.testem.instrumentationTiming.instrumentSlowly", args: ["{that}"] } + }, + false: false, + contextAwareness: { + coverage: { + checks: { + enableCoverage: { + contextValue: "{fluid.tests.testem.instrumentationTiming.harness.options.false}" + } + } + } } });