Retry mail test automation to address flakiness

.github/workflows/docker-test.yml

@@ -82,7 +82,7 @@ jobs:
           docker cp ${{ github.workspace }}/cypress/fixtures/mail-test.php ${{ matrix.dockerfile }}-msmtp:/var/www/baikal/html/
 
       - name: Run Cypress tests
-        run: npm run test -- --env MAILTRAP_TOKEN=${{ secrets.MAILTRAP_TOKEN }},MAILTRAP_ACCOUNTID=${{ secrets.MAILTRAP_ACCOUNTID }},MAILTRAP_INBOXID=${{ secrets.MAILTRAP_INBOXID }},MAILTRAP_SUBJECT='${{ github.sha }} - ${{ github.job }} attempt ${{ github.run_attempt }}' --config excludeSpecPattern='[]',screenshotsFolder=cypress/screenshots/mailtrap
+        run: npm run test -- --env MAILTRAP_TOKEN=${{ secrets.MAILTRAP_TOKEN }},MAILTRAP_ACCOUNTID=${{ secrets.MAILTRAP_ACCOUNTID }},MAILTRAP_INBOXID=${{ secrets.MAILTRAP_INBOXID }} --config excludeSpecPattern='[]',screenshotsFolder=cypress/screenshots/mailtrap
 
       - name: Archive test results
         if: always()

cypress/e2e/send-mail-with-php.mailtrap.cy.ts

@@ -1,54 +1,54 @@
 describe("Send mail with PHP", () => {
-  it("Should send an email with PHP", () => {
-    let token = Cypress.env("MAILTRAP_TOKEN");
-    let accountId = Cypress.env("MAILTRAP_ACCOUNTID");
-    let inboxId = Cypress.env("MAILTRAP_INBOXID");
-
-    expect(token).to.exist;
-    expect(accountId).to.exist;
-    expect(inboxId).to.exist;
-
-    // Clean email inbox
-    cy.request({
-      method: "PATCH",
-      url: `https://mailtrap.io/api/accounts/${accountId}/inboxes/${inboxId}/clean`,
-      auth: {
-        bearer: token,
+  it(
+    "Should send an email with PHP",
+    {
+      // Retry to address flakiness of sending and receiving emails
+      retries: {
+        runMode: 3,
+        openMode: 3,
       },
-    });
+    },
+    () => {
+      let token = Cypress.env("MAILTRAP_TOKEN");
+      let accountId = Cypress.env("MAILTRAP_ACCOUNTID");
+      let inboxId = Cypress.env("MAILTRAP_INBOXID");
 
-    // Send email through PHP
-    expect(Cypress.env("MAILTRAP_SUBJECT")).to.exist;
-    cy.request({
-      method: "GET",
-      url: "localhost/mail-test.php",
-      qs: {
-        subject: Cypress.env("MAILTRAP_SUBJECT"),
-      },
-    });
+      expect(token, "No mailtrap API token MAILTRAP_TOKEN").to.exist;
+      expect(accountId, "No mailtrap account id MAILTRAP_ACCOUNTID").to.exist;
+      expect(inboxId, "No mailtrap inbox id MAILTRAP_INBOXID").to.exist;
 
-    // Verify that the email arrived
-    cy.request({
-      method: "GET",
-      url: `https://mailtrap.io/api/accounts/${accountId}/inboxes/${inboxId}/messages`,
-      auth: {
-        bearer: token,
-      },
-    }).should((response) => {
-      expect(response.status).to.be.ok;
-      expect(response.body).to.be.an("array");
-      let actual = response.body.map((e) => ({
-        subject: e.subject,
-        from_email: e.from_email,
-        to_email: e.to_email,
-      }));
-      expect(actual).to.deep.include.members([
-        {
-          subject: Cypress.env("MAILTRAP_SUBJECT"),
-          from_email: "[email protected]",
-          to_email: "[email protected]",
+      // Send email through PHP
+      let subject = `Cypress ${crypto.randomUUID()}`;
+      cy.request({
+        method: "GET",
+        url: "localhost/mail-test.php",
+        qs: {
+          subject: subject,
+        },
+      });
+
+      // Verify that the email arrived
+      cy.request({
+        method: "GET",
+        url: `https://mailtrap.io/api/accounts/${accountId}/inboxes/${inboxId}/messages`,
+        auth: {
+          bearer: token,
         },
-      ]);
-    });
-  });
+      }).should((response) => {
+        expect(response.body).to.be.an("array");
+        let actual = response.body.map((e) => ({
+          subject: e.subject,
+          from_email: e.from_email,
+          to_email: e.to_email,
+        }));
+        expect(actual).to.deep.include.members([
+          {
+            subject: subject,
+            from_email: "[email protected]",
+            to_email: "[email protected]",
+          },
+        ]);
+      });
+    },
+  );
 });