diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index e2a7b62..0844b5f 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -117,7 +117,7 @@ jobs: run: yarn test:generate test-voiceover-preview: - name: Playwright VoiceOver Reduced Tests (${{ matrix.shardIndex }}) + name: Playwright VoiceOver Reduced Tests (${{ matrix.shardIndex }} / 1) needs: macos-install runs-on: macos-12 strategy: @@ -172,16 +172,16 @@ jobs: ./test-results ./recordings - - name: Archive Allure Artifacts + - name: Archive Blob Artifacts uses: actions/upload-artifact@v4 if: always() continue-on-error: true with: - name: allure-results-macos-voiceover-${{ matrix.shardIndex }} - path: ./allure-results + name: blob-report-macos-voiceover-${{ matrix.shardIndex }} + path: ./blob-report test-nvda-preview: - name: Playwright NVDA Reduced Tests (${{ matrix.shardIndex }}) + name: Playwright NVDA Reduced Tests (${{ matrix.shardIndex }} / 1) needs: windows-install runs-on: windows-2022 strategy: @@ -236,13 +236,13 @@ jobs: ./test-results ./recordings - - name: Archive Allure Artifacts + - name: Archive Blob Artifacts uses: actions/upload-artifact@v4 if: always() continue-on-error: true with: - name: allure-results-windows-nvda-${{ matrix.shardIndex }} - path: ./allure-results + name: blob-report-windows-nvda-${{ matrix.shardIndex }} + path: ./blob-report publish-preview-html-report: name: Publish Preview HTML Report To Vercel @@ -289,23 +289,9 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: allure-results-* - path: ./allure-results - - - name: Move Allure results into one directory - run: | - mkdir allure-results-final - for i in allure-results-macos-voiceover-{1..30}; do - if [ -d "$i" ]; then - mv $i/* allure-results-final/ - fi - done - for i in allure-results-windows-nvda-{1..30}; do - if [ -d "$i" ]; then - mv $i/* allure-results-final/ - fi - done - working-directory: ./allure-results + merge-multiple: true + pattern: blob-report-* + path: ./blob-report - name: Cache node_modules uses: actions/cache@v3 @@ -315,11 +301,11 @@ jobs: node_modules key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - name: Create Allure Test Report - run: npx allure generate ./allure-results/allure-results-final --clean -o ./allure-report + - name: Merge into HTML Report + run: npx playwright merge-reports --config=merge.config.ts ./blob-report - name: Cleanup Downloads - run: rm -rf ./allure-results || true + run: rm -rf ./blob-report || true - name: Deploy to Vercel uses: amondnet/vercel-action@v25 @@ -330,7 +316,7 @@ jobs: vercel-project-id: ${{ secrets.PREVIEW_VERCEL_PROJECT_ID }} github-comment: true github-token: ${{ secrets.GITHUB_TOKEN }} - working-directory: allure-report + working-directory: html-report - name: Update Deployment Status - Finish uses: bobheadxi/deployments@v1 diff --git a/.gitignore b/.gitignore index 73ff313..351d13e 100644 --- a/.gitignore +++ b/.gitignore @@ -111,7 +111,5 @@ playwright-report html-report testSuites.json blob-report -allure-report -allure-results .vercel diff --git a/macos.config.ts b/macos.config.ts index 0439484..b08fc6b 100644 --- a/macos.config.ts +++ b/macos.config.ts @@ -22,9 +22,7 @@ const config: PlaywrightTestConfig = { use: { ...devices["Desktop Chrome"], headless: false }, }, ], - reporter: process.env.CI - ? [["github"], ["allure-playwright", { outputDir: "allure-results" }]] - : "list", + reporter: process.env.CI ? [["github"], ["blob"]] : "list", }; export default config; diff --git a/merge.config.ts b/merge.config.ts new file mode 100644 index 0000000..6bc88cf --- /dev/null +++ b/merge.config.ts @@ -0,0 +1,4 @@ +export default { + testDir: "html-report", + reporter: [["html", { open: "never" }]], +}; diff --git a/package.json b/package.json index cd33b51..3b45eb3 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "aria-at", "playwright", "screen-reader", - "voiceover" + "voiceover", + "nvda" ], "scripts": { "ci": "yarn lint", @@ -40,8 +41,6 @@ "@types/node": "^20.10.5", "@typescript-eslint/eslint-plugin": "^6.16.0", "@typescript-eslint/parser": "^6.16.0", - "allure-commandline": "^2.25.0", - "allure-playwright": "^2.10.0", "eslint": "^8.56.0", "eslint-config-prettier": "^9.1.0", "ts-node": "^10.9.2", diff --git a/src/annotate.ts b/src/annotate.ts deleted file mode 100644 index 51277c6..0000000 --- a/src/annotate.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const annotate = ({ test, warning }) => { - console.log(warning); - - test.info().annotations.push({ - type: "issue", - description: warning, - }); -}; diff --git a/src/annotateIssue.ts b/src/annotateIssue.ts new file mode 100644 index 0000000..69dafdd --- /dev/null +++ b/src/annotateIssue.ts @@ -0,0 +1,8 @@ +export const annotateIssue = ({ test, issue }) => { + console.warn(issue); + + test.info().annotations.push({ + type: "issue", + description: issue, + }); +}; diff --git a/src/assert.ts b/src/assert.ts index 7128bc9..10e5290 100644 --- a/src/assert.ts +++ b/src/assert.ts @@ -1,7 +1,8 @@ import { ScreenReader } from "@guidepup/guidepup"; import { test as playwrightTest } from "@playwright/test"; import { expect } from "@playwright/test"; -import { annotate } from "./annotate"; +import { annotateIssue } from "./annotateIssue"; +import { log } from "./log"; /* * Explanation of the regular expression: @@ -106,9 +107,9 @@ export async function assert({ const phrase = (matches?.at(1) ?? matches?.at(2) ?? matches?.at(3))?.trim(); if (!phrase) { - annotate({ + annotateIssue({ test, - warning: `Unable to perform assertion: "${assertion}"`, + issue: `Unable to perform assertion: "${assertion}"`, }); continue; @@ -121,18 +122,18 @@ export async function assert({ const matchRegex = new RegExp(phraseRegex, "gi"); - console.log(`Performing assertion: "${assertion}"`); + log(`Performing assertion: "${assertion}"`); const found = !!spokenPhraseLog.find((spokenPhrase) => matchRegex.test(withRoleReplacements(spokenPhrase)) ); if (!found) { - console.log( + log( `Assertion "${assertion}" failed. Unable to find phrase "${phrase}" in spoken phrase log.` ); } else { - console.log(`Assertion "${assertion}" succeeded.`); + log(`Assertion "${assertion}" succeeded.`); } expect diff --git a/src/setAllureRecording.ts b/src/attachRecording.ts similarity index 54% rename from src/setAllureRecording.ts rename to src/attachRecording.ts index 535f847..e0a95c4 100644 --- a/src/setAllureRecording.ts +++ b/src/attachRecording.ts @@ -1,6 +1,6 @@ -import { accessSync, constants, readFileSync } from "fs"; +import { accessSync, constants } from "fs"; import { basename } from "path"; -import { allure } from "allure-playwright"; +import { test as playwrightTest } from "@playwright/test"; import { delay } from "./delay"; const MP4 = "video/mp4"; @@ -9,20 +9,22 @@ const MOV = "video/quicktime"; const EXISTS_RETRIES = 20; const EXISTS_WAIT = 100; -export const setAllureRecording = async ({ +export const attachRecording = async ({ osPlatform, - recordingFilePath, + path, + test, }: { osPlatform: NodeJS.Platform; - recordingFilePath: string; + path: string; + test: typeof playwrightTest; }) => { - if (!recordingFilePath) { + if (!path) { return; } for (let i = 0; i < EXISTS_RETRIES; i++) { try { - accessSync(recordingFilePath, constants.F_OK); + accessSync(path, constants.F_OK); break; } catch (e) { @@ -34,9 +36,8 @@ export const setAllureRecording = async ({ await delay(EXISTS_WAIT); } - const buffer = readFileSync(recordingFilePath); - const name = basename(recordingFilePath); + const name = basename(path); const contentType = osPlatform === "darwin" ? MOV : MP4; - await allure.attachment(name, buffer, contentType); + await test.info().attach(name, { contentType, path }); }; diff --git a/src/log.ts b/src/log.ts new file mode 100644 index 0000000..145e3d1 --- /dev/null +++ b/src/log.ts @@ -0,0 +1,2 @@ +export const log = (...messages) => + console.log(`[${new Date().toUTCString()}]`, ...messages); diff --git a/src/macOsVoiceOver.spec.ts b/src/macOsVoiceOver.spec.ts index cde5b9e..4907687 100644 --- a/src/macOsVoiceOver.spec.ts +++ b/src/macOsVoiceOver.spec.ts @@ -8,12 +8,13 @@ import { setup } from "./setup"; import { readTestSuitesCacheSync, TestSuite } from "./testSuites"; import { assert } from "./assert"; import { mapCommandToGuidepupKeys } from "./mapCommandToGuidepupKeys"; -import { annotate } from "./annotate"; +import { annotateIssue } from "./annotateIssue"; import { getTestDetails } from "./getTestDetails"; import { getScreenReaderTests } from "./getScreenReaderTests"; import { applicationNameMap } from "./applicationNameMap"; -import { setAllureMetadata } from "./setAllureMetadata"; -import { setAllureRecording } from "./setAllureRecording"; +import { setMetadata } from "./setMetadata"; +import { attachRecording } from "./attachRecording"; +import { log } from "./log"; // Allow sharding across describe blocks test.describe.configure({ mode: "parallel" }); @@ -96,16 +97,20 @@ const executeCommandSequence = async ({ const rawCommands = command.split(","); for (const rawCommand of rawCommands) { - console.log(`Performing command: "${rawCommand}".`); + log(`Performing command: "${rawCommand}".`); const { voiceOverCommand, mappedCommand, error } = mapCommand(rawCommand); if (error) { - annotate({ + const issue = `Unable to parse command: "${command}"`; + + annotateIssue({ test, - warning: `Unable to parse command: "${command}"`, + issue, }); + test.info().fixme(true, issue); + return; } @@ -126,7 +131,7 @@ const executeCommandSequence = async ({ const lastSpokenPhrase = await voiceOver.lastSpokenPhrase(); - console.log(`Screen reader output: "${lastSpokenPhrase}".`); + log(`Screen reader output: "${lastSpokenPhrase}".`); } }; @@ -173,7 +178,7 @@ const generateTestSuite = ({ screenReaderName, }); } catch { - annotate({ test, warning: "Screen recording failed." }); + annotateIssue({ test, issue: "Screen recording failed." }); } await setup({ @@ -194,27 +199,27 @@ const generateTestSuite = ({ browserName, voiceOver, }) => { - await setAllureMetadata({ + await setMetadata({ browserName, browserVersion: browser.version(), osPlatform, osRelease, references, screenReaderTest, + test, testUrl, }); await executeCommandSequence({ browserName, command, voiceOver }); await assert({ assertions, screenReader: voiceOver, test }); try { - const recordingFilePath = stopRecording?.(); - - await setAllureRecording({ + await attachRecording({ osPlatform, - recordingFilePath, + path: stopRecording?.(), + test, }); } catch { - annotate({ test, warning: "Screen recording failed." }); + annotateIssue({ test, issue: "Screen recording failed." }); } }); } diff --git a/src/nvdaTest.ts b/src/nvdaTest.ts index 4d381f5..ae35298 100644 --- a/src/nvdaTest.ts +++ b/src/nvdaTest.ts @@ -38,6 +38,8 @@ const nvdaTest = test.extend<{ nvda: typeof nvda }>({ } if (nvdaStartRetryCount === 3) { + test.info().fixme(true, error.message); + throw error; } } diff --git a/src/setAllureMetadata.ts b/src/setAllureMetadata.ts deleted file mode 100644 index dd7222d..0000000 --- a/src/setAllureMetadata.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { allure } from "allure-playwright"; -import { References, Test } from "./types"; - -export const setAllureMetadata = async ({ - browserName, - browserVersion, - osPlatform, - osRelease, - references, - screenReaderTest, - testUrl, -}: { - browserName: string; - browserVersion: string; - osPlatform: NodeJS.Platform; - osRelease: string; - references: References; - screenReaderTest: Test; - testUrl: string; -}) => { - const { - author, - authorEmail, - title: epic, - designPattern, - example, - ...additionalReferences - } = references; - - const { title: story, task, instructions } = screenReaderTest; - - await allure.epic(epic); - await allure.story(story); - - await allure.parameter("browserName", browserName); - await allure.parameter("browserVersion", browserVersion); - await allure.parameter("osPlatform", osPlatform); - await allure.parameter("osRelease", osRelease); - - await allure.parameter("author", author); - await allure.parameter("authorEmail", authorEmail); - await allure.parameter("task", task); - await allure.parameter("instructions", instructions); - - await allure.link(testUrl, "reference"); - await allure.link(designPattern, "designPattern"); - await allure.link(example, "example"); - - for (const [key, value] of Object.entries(additionalReferences)) { - if (value.startsWith("https://")) { - await allure.link(value, key); - } - } -}; diff --git a/src/setMetadata.ts b/src/setMetadata.ts new file mode 100644 index 0000000..a1afb8b --- /dev/null +++ b/src/setMetadata.ts @@ -0,0 +1,78 @@ +import { test as playwrightTest } from "@playwright/test"; +import { References, Test } from "./types"; + +export const setMetadata = async ({ + browserName, + browserVersion, + osPlatform, + osRelease, + references, + screenReaderTest, + test, + testUrl, +}: { + browserName: string; + browserVersion: string; + osPlatform: NodeJS.Platform; + osRelease: string; + references: References; + screenReaderTest: Test; + test: typeof playwrightTest; + testUrl: string; +}) => { + const { + title: referenceTitle, + // Use the `testUrl` instead as more useful. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + reference: _, + ...additionalReferences + } = references; + + const { title: testTitle, task, instructions } = screenReaderTest; + + test.info().annotations.push( + { + type: "referenceTitle", + description: referenceTitle, + }, + { + type: "testTitle", + description: testTitle, + }, + { + type: "browserName", + description: browserName, + }, + { + type: "browserVersion", + description: browserVersion, + }, + { + type: "osPlatform", + description: osPlatform, + }, + { + type: "osRelease", + description: osRelease, + }, + { + type: "task", + description: task, + }, + { + type: "instructions", + description: instructions, + }, + { + type: "reference", + description: testUrl, + } + ); + + for (const [key, value] of Object.entries(additionalReferences)) { + test.info().annotations.push({ + type: key, + description: value, + }); + } +}; diff --git a/src/setup.ts b/src/setup.ts index b6f8e94..0e3c16f 100644 --- a/src/setup.ts +++ b/src/setup.ts @@ -2,6 +2,7 @@ import { Page } from "@playwright/test"; import { delay } from "./delay"; import { ScreenReader } from "@guidepup/guidepup"; import { Test } from "./types"; +import { log } from "./log"; const PAGE_LOAD_DELAY = 250; @@ -28,7 +29,7 @@ export async function setup({ }) => void | Promise; testUrl: string; }): Promise { - console.log(`Navigating to URL: "${testUrl}".`); + log(`Navigating to URL: "${testUrl}".`); await page.goto(testUrl, { waitUntil: "load" }); await delay(PAGE_LOAD_DELAY); @@ -38,16 +39,17 @@ export async function setup({ await screenReader.clearItemTextLog(); if (hasSetupScript) { - console.log(`Running test setup.`); + log("Running test setup."); await screenReader.act(); const lastSpokenPhrase = await screenReader.lastSpokenPhrase(); - console.log(`Screen reader output: "${lastSpokenPhrase}".`); - console.log(`Running test steps.`); + log(`Screen reader output: "${lastSpokenPhrase}".`); + log("Running test steps."); } else { - console.log(`No test setup configured. Running test steps.`); + log("No test setup configured."); + log("Running test steps."); } await setMode({ mode, screenReader }); diff --git a/src/windowsNvda.spec.ts b/src/windowsNvda.spec.ts index 2341bbd..1341023 100644 --- a/src/windowsNvda.spec.ts +++ b/src/windowsNvda.spec.ts @@ -8,12 +8,13 @@ import { setup } from "./setup"; import { readTestSuitesCacheSync, TestSuite } from "./testSuites"; import { assert } from "./assert"; import { mapCommandToGuidepupKeys } from "./mapCommandToGuidepupKeys"; -import { annotate } from "./annotate"; +import { annotateIssue } from "./annotateIssue"; import { getTestDetails } from "./getTestDetails"; import { getScreenReaderTests } from "./getScreenReaderTests"; -import { setAllureMetadata } from "./setAllureMetadata"; -import { setAllureRecording } from "./setAllureRecording"; +import { setMetadata } from "./setMetadata"; +import { attachRecording } from "./attachRecording"; import { Test } from "./types"; +import { log } from "./log"; // Allow sharding across describe blocks test.describe.configure({ mode: "parallel" }); @@ -83,14 +84,18 @@ const executeCommandSequence = async ({ for (const rawCommand of rawCommands) { const { mappedCommand, error } = mapCommand(rawCommand); - console.log(`Performing command: "${rawCommand}"`); + log(`Performing command: "${rawCommand}"`); if (error) { - annotate({ + const issue = `Unable to parse command: "${command}"`; + + annotateIssue({ test, - warning: `Unable to parse command: "${command}"`, + issue, }); + test.info().fixme(true, issue); + return; } @@ -98,7 +103,7 @@ const executeCommandSequence = async ({ const lastSpokenPhrase = await nvda.lastSpokenPhrase(); - console.log(`Screen reader output: "${lastSpokenPhrase}".`); + log(`Screen reader output: "${lastSpokenPhrase}".`); } }; @@ -145,7 +150,7 @@ const generateTestSuite = ({ screenReaderName, }); } catch { - annotate({ test, warning: "Screen recording failed." }); + annotateIssue({ test, issue: "Screen recording failed." }); } await setup({ @@ -165,27 +170,27 @@ const generateTestSuite = ({ browserName, nvda, }) => { - await setAllureMetadata({ + await setMetadata({ browserName, browserVersion: browser.version(), osPlatform, osRelease, references, screenReaderTest, + test, testUrl, }); await executeCommandSequence({ command, nvda }); await assert({ assertions, screenReader: nvda, test }); try { - const recordingFilePath = stopRecording?.(); - - await setAllureRecording({ + await attachRecording({ osPlatform, - recordingFilePath, + path: stopRecording?.(), + test, }); } catch { - annotate({ test, warning: "Screen recording failed." }); + annotateIssue({ test, issue: "Screen recording failed." }); } }); } diff --git a/windows.config.ts b/windows.config.ts index 1574e06..7f5990d 100644 --- a/windows.config.ts +++ b/windows.config.ts @@ -18,9 +18,7 @@ const config: PlaywrightTestConfig = { use: { ...devices["Desktop Chrome"], headless: false }, }, ], - reporter: process.env.CI - ? [["github"], ["allure-playwright", { outputDir: "allure-results" }]] - : "list", + reporter: process.env.CI ? [["github"], ["blob"]] : "list", }; export default config; diff --git a/yarn.lock b/yarn.lock index c57a45c..044fbee 100644 --- a/yarn.lock +++ b/yarn.lock @@ -308,25 +308,6 @@ ajv@^6.12.4: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -allure-commandline@^2.25.0: - version "2.25.0" - resolved "https://registry.yarnpkg.com/allure-commandline/-/allure-commandline-2.25.0.tgz#46a42b91a5fdeb70a4d2a077bed93515b1c7dd65" - integrity sha512-HYMQwuX/8ySTrXqgr6ZY830c6xgRbSusxVwZIBbDyNTCDb3mqK2b1ENjsD8yhl/7/8S02NZCDAwJFxt3f8qL9Q== - -allure-js-commons@2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/allure-js-commons/-/allure-js-commons-2.10.0.tgz#59edd18ac283bd58d848e47fc7e625342921a43a" - integrity sha512-DgACWBU2dchQD8tQOo5Y0MXx08SSzdgCnKBdwrOu29vITYBXih+0r8SbmrFYQhjAbn8eKMM+mXq+rKtjZRa2oA== - dependencies: - properties "^1.2.1" - -allure-playwright@^2.10.0: - version "2.10.0" - resolved "https://registry.yarnpkg.com/allure-playwright/-/allure-playwright-2.10.0.tgz#f5ff5a1dcc946ed92b056a1d45d7aa3c55b391ca" - integrity sha512-zDljPJ/Fnyd2fn7msChtZebwkSVmAGLe/oWK7okGi0Ed+iHZ0E5Vwe5Z5MtUdfLjnT/OkOduwmS7R/DHeTXFSA== - dependencies: - allure-js-commons "2.10.0" - ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -1055,11 +1036,6 @@ progress@^2.0.3: resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== -properties@^1.2.1: - version "1.2.1" - resolved "https://registry.yarnpkg.com/properties/-/properties-1.2.1.tgz#0ee97a7fc020b1a2a55b8659eda4aa8d869094bd" - integrity sha512-qYNxyMj1JeW54i/EWEFsM1cVwxJbtgPp8+0Wg9XjNaK6VE/c4oRi6PNu5p7w1mNXEIQIjV5Wwn8v8Gz82/QzdQ== - punycode@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"