More easily track link checks that take a long time

jc-verse · Aug 6, 2024 · d67799f · d67799f
1 parent 6fc7872
commit d67799f
Show file tree

Hide file tree

Showing 6 changed files with 32 additions and 9 deletions.
diff --git a/config/allowed-code-link-text.txt b/config/allowed-code-link-text.txt
@@ -1,3 +1,6 @@
+  Texts in this file will not be reported by the "Code with space" or "Code with underscore" checks.
+  Use two-space indent for comments.
+
   CSS syntax:
 / <counter>
 / <string>

diff --git a/config/inaccessible-links.txt b/config/inaccessible-links.txt
@@ -1,3 +1,6 @@
+  Links in this file will be ignored by the link checker, usually because they are behind auth or crawler checks.
+  Use two-space indent for comments.
+
   Cloudflare protection (response status 403 and returned HTML contains "Just a moment..."):
 https://codepen.io{/?}
 https://gitlab.com/projects/new
@@ -16,3 +19,13 @@ https://www.openwebanalytics.com{/?}
 https://www.techopedia.com/definition/{.*}
 https://www.webpagetest.org{/?}
 https://www.reddit.com/r/{.*}
+
+  Goes to login:
+https://cloud.mongodb.com/v2
+https://console.cloud.google.com/{.*}
+https://docs.google.com/drawings
+https://shell.cloud.google.com/{.*}
+https://sites.google.com/{.*}
+https://github.com/new
+https://github.com/{.*}/issues/new{.*}
+https://github.com/orgs/mdn/teams{.*}
diff --git a/config/missing-features.txt b/config/missing-features.txt
@@ -1,3 +1,6 @@
+  This file is autogenrated. Run `console.log([...document.querySelectorAll(".features a")].map((x) => x.innerText).join("\n"))` on https://openwebdocs.github.io/web-docs-backlog/all/ to get the new list.
+  Use two-space indent for comments.
+
 api.AbortPaymentEvent
 api.AbortPaymentEvent.AbortPaymentEvent
 api.AbortPaymentEvent.respondWith

diff --git a/config/no-page.txt b/config/no-page.txt
@@ -1,3 +1,8 @@
+  Links in this file will be ignored by the "Broken link" and "Flaw broken_links" checks.
+  They represent pages that should be created. If a page below should not be created,
+  it should be removed from this file so the broken link can be fixed.
+  Use two-space indent for comments.
+
 /en-US/docs/Glossary/IME
 /en-US/docs/Glossary/IP
 /en-US/docs/Glossary/SMIL
@@ -19,7 +24,6 @@
 /en-US/docs/Web/API/BluetoothRemoteGATTCharacteristic/characteristicvaluechanged_event
 /en-US/docs/Web/API/CSSMarginRule
 /en-US/docs/Web/API/CSSNumericType
-/en-US/docs/Web/API/CSSStyleDeclaration/named_properties
 /en-US/docs/Web/API/DOMMatrix/fromFloat32Array
 /en-US/docs/Web/API/DOMMatrix/fromFloat64Array
 /en-US/docs/Web/API/DOMMatrix/fromMatrix
@@ -96,7 +100,6 @@
 /en-US/docs/Web/API/RTCReceivedRtpStreamStats/framesDecoded
 /en-US/docs/Web/API/RTCReceiverAudioTrackAttachmentStats
 /en-US/docs/Web/API/RTCReceiverVideoTrackAttachmentStats
-/en-US/docs/Web/API/RTCRemoteInboundRtpStreamStats
 /en-US/docs/Web/API/RTCRtcpParameters
 /en-US/docs/Web/API/RTCSenderAudioTrackAttachmentStats
 /en-US/docs/Web/API/RTCSenderVideoTrackAttachmentStats

diff --git a/src/server/create-graph.ts b/src/server/create-graph.ts
@@ -27,7 +27,7 @@ const allowedSpacedCodeLink = [
   // HTTP status
   /^\d+ [\w '-]+$/,
   // HTTP header
-  /^(Cache-Control|Clear-Site-Data|Connection|Content-Security-Policy|Cross-Origin-Opener-Policy|Cross-Origin-Resource-Policy|Permissions-Policy|Sec-Purpose): ([\w-]+|"[\w-]+")$/,
+  /^(Cache-Control|Clear-Site-Data|Connection|Content-Length|Content-Security-Policy|Cross-Origin-Opener-Policy|Cross-Origin-Resource-Policy|Permissions-Policy|Sec-Purpose|Transfer-Encoding): ([\w-]+|"[\w-]+")$/,
   // MIME
   /^[a-z]+\/[\w+-]+; [a-z]+=("[\w ,.-]+"|\w+);?$/,
   // Macro calls

diff --git a/src/server/process-warnings.ts b/src/server/process-warnings.ts
@@ -280,19 +280,20 @@ async function depleteQueue() {
     promisePool.push(linkRequests[i][1]().then(() => i));
   }
   while (curReq < linkRequests.length) {
-    if (curReq % 100 === 0 || linkRequests.length - curReq < 100) {
-      console.log(`Processed ${curReq - queueLen}/${linkRequests.length} links`);
-    }
     const completedSlot = await Promise.race(promisePool);
     linksInPool[completedSlot] = linkRequests[curReq][0];
     promisePool[completedSlot] = linkRequests[curReq][1]().then(
       () => completedSlot
     );
     curReq++;
+    if (curReq % 100 === queueLen || linkRequests.length - curReq < 100) {
+      console.log(`Processed ${curReq - queueLen}/${linkRequests.length} links`);
+    }
+  }
+  for (let i = 1; i <= queueLen; i++) {
+    const completedSlot = await Promise.race(promisePool);
+    console.log(`Processed ${curReq - queueLen + i}/${linkRequests.length} links (${linksInPool[completedSlot]})`);
   }
-  console.log(`Waiting for the last requests to finish:\n${linksInPool.join("\n")}`);
-  await Promise.all(promisePool);
-  console.log(`Processed ${curReq}/${linkRequests.length} links`);
 }
 
 if (!Bun.argv.includes("--no-external-link-check")) {