feat: move away from allure

.github/workflows/preview.yml

@@ -117,7 +117,7 @@ jobs:
         run: yarn test:generate
 
   test-voiceover-preview:
-    name: Playwright VoiceOver Reduced Tests (${{ matrix.shardIndex }})
+    name: Playwright VoiceOver Reduced Tests (${{ matrix.shardIndex }} / 1)
     needs: macos-install
     runs-on: macos-12
     strategy:
@@ -172,16 +172,16 @@ jobs:
             ./test-results
             ./recordings
 
-      - name: Archive Allure Artifacts
+      - name: Archive Blob Artifacts
         uses: actions/upload-artifact@v4
         if: always()
         continue-on-error: true
         with:
-          name: allure-results-macos-voiceover-${{ matrix.shardIndex }}
-          path: ./allure-results
+          name: blob-report-macos-voiceover-${{ matrix.shardIndex }}
+          path: ./blob-report
 
   test-nvda-preview:
-    name: Playwright NVDA Reduced Tests (${{ matrix.shardIndex }})
+    name: Playwright NVDA Reduced Tests (${{ matrix.shardIndex }} / 1)
     needs: windows-install
     runs-on: windows-2022
     strategy:
@@ -236,13 +236,13 @@ jobs:
             ./test-results
             ./recordings
 
-      - name: Archive Allure Artifacts
+      - name: Archive Blob Artifacts
         uses: actions/upload-artifact@v4
         if: always()
         continue-on-error: true
         with:
-          name: allure-results-windows-nvda-${{ matrix.shardIndex }}
-          path: ./allure-results
+          name: blob-report-windows-nvda-${{ matrix.shardIndex }}
+          path: ./blob-report
 
   publish-preview-html-report:
     name: Publish Preview HTML Report To Vercel
@@ -289,23 +289,9 @@ jobs:
 
       - uses: actions/download-artifact@v4
         with:
-          pattern: allure-results-*
-          path: ./allure-results
-
-      - name: Move Allure results into one directory
-        run: |
-          mkdir allure-results-final
-          for i in allure-results-macos-voiceover-{1..30}; do
-            if [ -d "$i" ]; then
-              mv $i/* allure-results-final/
-            fi
-          done
-          for i in allure-results-windows-nvda-{1..30}; do
-            if [ -d "$i" ]; then
-              mv $i/* allure-results-final/
-            fi
-          done
-        working-directory: ./allure-results
+          merge-multiple: true
+          pattern: blob-report-*
+          path: ./blob-report
 
       - name: Cache node_modules
         uses: actions/cache@v3
@@ -315,11 +301,11 @@ jobs:
             node_modules
           key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }}
 
-      - name: Create Allure Test Report
-        run: npx allure generate ./allure-results/allure-results-final --clean -o ./allure-report
+      - name: Merge into HTML Report
+        run: npx playwright merge-reports --config=merge.config.ts ./blob-report
 
       - name: Cleanup Downloads
-        run: rm -rf ./allure-results || true
+        run: rm -rf ./blob-report || true
 
       - name: Deploy to Vercel
         uses: amondnet/vercel-action@v25
@@ -330,7 +316,7 @@ jobs:
           vercel-project-id: ${{ secrets.PREVIEW_VERCEL_PROJECT_ID }}
           github-comment: true
           github-token: ${{ secrets.GITHUB_TOKEN }}
-          working-directory: allure-report
+          working-directory: html-report
 
       - name: Update Deployment Status - Finish
         uses: bobheadxi/deployments@v1

.gitignore

@@ -111,7 +111,5 @@ playwright-report
 html-report
 testSuites.json
 blob-report
-allure-report
-allure-results
 
 .vercel

macos.config.ts

@@ -22,9 +22,7 @@ const config: PlaywrightTestConfig = {
       use: { ...devices["Desktop Chrome"], headless: false },
     },
   ],
-  reporter: process.env.CI
-    ? [["github"], ["allure-playwright", { outputDir: "allure-results" }]]
-    : "list",
+  reporter: process.env.CI ? [["github"], ["blob"]] : "list",
 };
 
 export default config;

merge.config.ts

@@ -0,0 +1,4 @@
+export default {
+  testDir: "html-report",
+  reporter: [["html", { open: "never" }]],
+};

package.json

@@ -19,7 +19,8 @@
     "aria-at",
     "playwright",
     "screen-reader",
-    "voiceover"
+    "voiceover",
+    "nvda"
   ],
   "scripts": {
     "ci": "yarn lint",
@@ -40,8 +41,6 @@
     "@types/node": "^20.10.5",
     "@typescript-eslint/eslint-plugin": "^6.16.0",
     "@typescript-eslint/parser": "^6.16.0",
-    "allure-commandline": "^2.25.0",
-    "allure-playwright": "^2.10.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",
     "ts-node": "^10.9.2",

src/annotateIssue.ts

@@ -0,0 +1,8 @@
+export const annotateIssue = ({ test, issue }) => {
+  console.warn(issue);
+
+  test.info().annotations.push({
+    type: "issue",
+    description: issue,
+  });
+};

src/assert.ts

@@ -1,7 +1,8 @@
 import { ScreenReader } from "@guidepup/guidepup";
 import { test as playwrightTest } from "@playwright/test";
 import { expect } from "@playwright/test";
-import { annotate } from "./annotate";
+import { annotateIssue } from "./annotateIssue";
+import { log } from "./log";
 
 /*
  * Explanation of the regular expression:
@@ -106,9 +107,9 @@ export async function assert({
     const phrase = (matches?.at(1) ?? matches?.at(2) ?? matches?.at(3))?.trim();
 
     if (!phrase) {
-      annotate({
+      annotateIssue({
         test,
-        warning: `Unable to perform assertion: "${assertion}"`,
+        issue: `Unable to perform assertion: "${assertion}"`,
       });
 
       continue;
@@ -121,18 +122,18 @@ export async function assert({
 
     const matchRegex = new RegExp(phraseRegex, "gi");
 
-    console.log(`Performing assertion: "${assertion}"`);
+    log(`Performing assertion: "${assertion}"`);
 
     const found = !!spokenPhraseLog.find((spokenPhrase) =>
       matchRegex.test(withRoleReplacements(spokenPhrase))
     );
 
     if (!found) {
-      console.log(
+      log(
         `Assertion "${assertion}" failed. Unable to find phrase "${phrase}" in spoken phrase log.`
       );
     } else {
-      console.log(`Assertion "${assertion}" succeeded.`);
+      log(`Assertion "${assertion}" succeeded.`);
     }
 
     expect

src/setAllureRecording.ts → src/attachRecording.ts

@@ -1,6 +1,6 @@
-import { accessSync, constants, readFileSync } from "fs";
+import { accessSync, constants } from "fs";
 import { basename } from "path";
-import { allure } from "allure-playwright";
+import { test as playwrightTest } from "@playwright/test";
 import { delay } from "./delay";
 
 const MP4 = "video/mp4";
@@ -9,20 +9,22 @@ const MOV = "video/quicktime";
 const EXISTS_RETRIES = 20;
 const EXISTS_WAIT = 100;
 
-export const setAllureRecording = async ({
+export const attachRecording = async ({
   osPlatform,
-  recordingFilePath,
+  path,
+  test,
 }: {
   osPlatform: NodeJS.Platform;
-  recordingFilePath: string;
+  path: string;
+  test: typeof playwrightTest;
 }) => {
-  if (!recordingFilePath) {
+  if (!path) {
     return;
   }
 
   for (let i = 0; i < EXISTS_RETRIES; i++) {
     try {
-      accessSync(recordingFilePath, constants.F_OK);
+      accessSync(path, constants.F_OK);
 
       break;
     } catch (e) {
@@ -34,9 +36,8 @@ export const setAllureRecording = async ({
     await delay(EXISTS_WAIT);
   }
 
-  const buffer = readFileSync(recordingFilePath);
-  const name = basename(recordingFilePath);
+  const name = basename(path);
   const contentType = osPlatform === "darwin" ? MOV : MP4;
 
-  await allure.attachment(name, buffer, contentType);
+  await test.info().attach(name, { contentType, path });
 };

src/log.ts

@@ -0,0 +1,2 @@
+export const log = (...messages) =>
+  console.log(`[${new Date().toUTCString()}]`, ...messages);

src/macOsVoiceOver.spec.ts

@@ -8,12 +8,13 @@ import { setup } from "./setup";
 import { readTestSuitesCacheSync, TestSuite } from "./testSuites";
 import { assert } from "./assert";
 import { mapCommandToGuidepupKeys } from "./mapCommandToGuidepupKeys";
-import { annotate } from "./annotate";
+import { annotateIssue } from "./annotateIssue";
 import { getTestDetails } from "./getTestDetails";
 import { getScreenReaderTests } from "./getScreenReaderTests";
 import { applicationNameMap } from "./applicationNameMap";
-import { setAllureMetadata } from "./setAllureMetadata";
-import { setAllureRecording } from "./setAllureRecording";
+import { setMetadata } from "./setMetadata";
+import { attachRecording } from "./attachRecording";
+import { log } from "./log";
 
 // Allow sharding across describe blocks
 test.describe.configure({ mode: "parallel" });
@@ -96,16 +97,20 @@ const executeCommandSequence = async ({
   const rawCommands = command.split(",");
 
   for (const rawCommand of rawCommands) {
-    console.log(`Performing command: "${rawCommand}".`);
+    log(`Performing command: "${rawCommand}".`);
 
     const { voiceOverCommand, mappedCommand, error } = mapCommand(rawCommand);
 
     if (error) {
-      annotate({
+      const issue = `Unable to parse command: "${command}"`;
+
+      annotateIssue({
         test,
-        warning: `Unable to parse command: "${command}"`,
+        issue,
       });
 
+      test.info().fixme(true, issue);
+
       return;
     }
 
@@ -126,7 +131,7 @@ const executeCommandSequence = async ({
 
     const lastSpokenPhrase = await voiceOver.lastSpokenPhrase();
 
-    console.log(`Screen reader output: "${lastSpokenPhrase}".`);
+    log(`Screen reader output: "${lastSpokenPhrase}".`);
   }
 };
 
@@ -173,7 +178,7 @@ const generateTestSuite = ({
               screenReaderName,
             });
           } catch {
-            annotate({ test, warning: "Screen recording failed." });
+            annotateIssue({ test, issue: "Screen recording failed." });
           }
 
           await setup({
@@ -194,27 +199,27 @@ const generateTestSuite = ({
             browserName,
             voiceOver,
           }) => {
-            await setAllureMetadata({
+            await setMetadata({
               browserName,
               browserVersion: browser.version(),
               osPlatform,
               osRelease,
               references,
               screenReaderTest,
+              test,
               testUrl,
             });
             await executeCommandSequence({ browserName, command, voiceOver });
             await assert({ assertions, screenReader: voiceOver, test });
 
             try {
-              const recordingFilePath = stopRecording?.();
-
-              await setAllureRecording({
+              await attachRecording({
                 osPlatform,
-                recordingFilePath,
+                path: stopRecording?.(),
+                test,
               });
             } catch {
-              annotate({ test, warning: "Screen recording failed." });
+              annotateIssue({ test, issue: "Screen recording failed." });
             }
           });
         }

src/nvdaTest.ts

@@ -38,6 +38,8 @@ const nvdaTest = test.extend<{ nvda: typeof nvda }>({
         }
 
         if (nvdaStartRetryCount === 3) {
+          test.info().fixme(true, error.message);
+
           throw error;
         }
       }