Show more queries that are too long.

2 issues this solves

1. Show all the failures instead of only the first failure

   This was an issue because I'd fix the one failure only to later
   find there were worse cases. Just the worst case is also not
   enough.

2. Show a representation of each test

   The issue where is whatever is printing the `Error` message
   truncates the list so when I printed them all and there were
   more than about 20 because of so many permutations it would
   cut off the list, defeating the point of (1) above.

   So, opted to show one query per test of each length.

Example:

            Error: Generated test variant would produce too-long -actual.txt filename. Possible solutions:
              - Reduce the length of the parts of the test query
              - Reduce the parameterization of the test
              - Make the test function faster and regenerate the listing_meta entry
              - Reduce the specificity of test expectations (if you're using them)
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="compute";format="astc-8x8-unorm-srgb";dim="cube";filt="linear";modeU="m";modeV="m";modeW="m";*

            Error: Generated test variant would produce too-long -actual.txt filename. Possible solutions:
              - Reduce the length of the parts of the test query
              - Reduce the parameterization of the test
              - Make the test function faster and regenerate the listing_meta entry
              - Reduce the specificity of test expectations (if you're using them)
            |<------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------>|
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="compute";format="astc-8x8-unorm-srgb";dim="cube";filt="linear";modeU="m";modeV="m";modeW="m";*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="compute";format="astc-12x12-unorm-srgb";dim="cube";filt="linear";modeU="m";modeV="m";modeW="m";*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="compute";format="astc-10x8-unorm-srgb";dim="cube";filt="linear";modeU="m";modeV="m";modeW="m";*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="fragment";format="astc-12x12-unorm-srgb";dim="cube";filt="nearest";modeU="m";modeV="m";modeW="m";*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_3d_coords:stage="compute";format="astc-12x12-unorm-srgb";dim="cube";filt="nearest";modeU="m";modeV="m";modeW="m";*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_array_2d_coords:stage="compute";format="astc-12x12-unorm-srgb";filt="linear";modeU="m";modeV="m";offset=false;*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_array_2d_coords:stage="fragment";format="astc-12x12-unorm-srgb";filt="nearest";modeU="m";modeV="m";offset=false;*
            webgpu:shader,execution,expression,call,builtin,textureSampleGrad:sampled_array_2d_coords:stage="compute";format="astc-12x12-unorm-srgb";filt="nearest";modeU="m";modeV="m";offset=false;*
            webgpu:shader,execution,expression,call,builtin,textureSampleLevel:sampled_array_2d_coords:stage="fragment";format="astc-12x12-unorm-srgb";filt="nearest";modeU="m";modeV="m";offset=false
            webgpu:shader,execution,expression,call,builtin,textureSampleLevel:sampled_array_2d_coords:stage="compute";format="astc-12x12-unorm-srgb";filt="nearest";modeU="m";modeV="m";offset=false

src/common/tools/gen_wpt_cts_html.ts

@@ -9,6 +9,8 @@ import {
 } from '../internal/query/query.js';
 import { assert } from '../util/util.js';
 
+const kMaxQueryLength = 184;
+
 function printUsageAndExit(rc: number): never {
   console.error(`\
 Usage (simple, for webgpu:* suite only):
@@ -193,6 +195,7 @@ let config: Config;
 
   const loader = new DefaultTestFileLoader();
   const lines = [];
+  const tooLongQueries = [];
   for (const prefix of config.argumentsPrefixes) {
     const rootQuery = new TestQueryMultiFile(config.suite, []);
     const tree = await loader.loadTree(rootQuery, {
@@ -219,15 +222,9 @@ let config: Config;
         // Check for a safe-ish path length limit. Filename must be <= 255, and on Windows the whole
         // path must be <= 259. Leave room for e.g.:
         // 'c:\b\s\w\xxxxxxxx\layout-test-results\external\wpt\webgpu\cts_worker=0_q=...-actual.txt'
-        assert(
-          queryString.length < 185,
-          `Generated test variant would produce too-long -actual.txt filename. Possible solutions:
-- Reduce the length of the parts of the test query
-- Reduce the parameterization of the test
-- Make the test function faster and regenerate the listing_meta entry
-- Reduce the specificity of test expectations (if you're using them)
-${queryString}`
-        );
+        if (queryString.length > kMaxQueryLength) {
+          tooLongQueries.push(queryString);
+        }
       }
 
       lines.push({
@@ -243,6 +240,29 @@ ${queryString}`
     }
     prefixComment.comment += `; ${variantCount} variants generated from ${testsSeen.size} tests in ${filesSeen.size} files`;
   }
+
+  if (tooLongQueries.length > 0) {
+    // Try to show some representation of failures. We show one entry from each
+    // test that is different length. Without this the logger cuts off the error
+    // messages and you end up not being told about which tests have issues.
+    const queryStrings = new Map<string, string>();
+    tooLongQueries.forEach(s => {
+      const colonNdx = s.lastIndexOf(':');
+      const prefix = s.substring(0, colonNdx + 1);
+      const id = `${prefix}:${s.length}`;
+      queryStrings.set(id, s);
+    });
+    throw new Error(
+      `Generated test variant would produce too-long -actual.txt filename. Possible solutions:
+  - Reduce the length of the parts of the test query
+  - Reduce the parameterization of the test
+  - Make the test function faster and regenerate the listing_meta entry
+  - Reduce the specificity of test expectations (if you're using them)
+|<${''.padEnd(kMaxQueryLength - 4, '-')}>|
+${[...queryStrings.values()].join('\n')}`
+    );
+  }
+
   await generateFile(lines);
 })().catch(ex => {
   console.log(ex.stack ?? ex.toString());