diff --git a/.eslintrc.json b/.eslintrc.json index 96d88695f3a1..14c03a55e481 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -52,7 +52,10 @@ "@typescript-eslint/no-this-alias": "warn", "@typescript-eslint/no-unnecessary-type-assertion": "warn", "@typescript-eslint/no-unnecessary-type-constraint": "warn", - "@typescript-eslint/no-unused-vars": ["warn", { "vars": "all", "args": "none" }], + "@typescript-eslint/no-unused-vars": [ + "warn", + { "vars": "all", "args": "none", "varsIgnorePattern": "^_" } + ], "@typescript-eslint/prefer-as-const": "warn", "@typescript-eslint/prefer-for-of": "warn", "@typescript-eslint/prefer-namespace-keyword": "warn", diff --git a/Gruntfile.js b/Gruntfile.js index 03f5d1cfcc38..9bb14e2f14c2 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -26,13 +26,21 @@ module.exports = function (grunt) { cmd: 'node', args: ['tools/validate', 'src/webgpu', 'src/stress', 'src/manual', 'src/unittests', 'src/demo'], }, + 'validate-cache': { + cmd: 'node', + args: ['tools/gen_cache', 'out', 'src/webgpu', '--validate'], + }, 'generate-wpt-cts-html': { cmd: 'node', - args: ['tools/gen_wpt_cts_html', 'out-wpt/cts.https.html', 'src/common/templates/cts.https.html'], + args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_unchunked.json'], + }, + 'generate-wpt-cts-html-chunked2sec': { + cmd: 'node', + args: ['tools/gen_wpt_cts_html', 'tools/gen_wpt_cfg_chunked2sec.json'], }, 'generate-cache': { cmd: 'node', - args: ['tools/gen_cache', 'out/data', 'src/webgpu'], + args: ['tools/gen_cache', 'out', 'src/webgpu'], }, unittest: { cmd: 'node', @@ -181,6 +189,7 @@ module.exports = function (grunt) { 'copy:out-wpt-generated', 'copy:out-wpt-htmlfiles', 'run:generate-wpt-cts-html', + 'run:generate-wpt-cts-html-chunked2sec', ]); grunt.registerTask('build-done-message', () => { process.stderr.write('\nBuild completed! Running checks/tests'); @@ -189,6 +198,7 @@ module.exports = function (grunt) { registerTaskAndAddToHelp('pre', 'Run all presubmit checks: standalone+wpt+typecheck+unittest+lint', [ 'clean', 'run:validate', + 'run:validate-cache', 'build-standalone', 'run:generate-listings', 'build-wpt', diff --git a/docs/adding_timing_metadata.md b/docs/adding_timing_metadata.md new file mode 100644 index 000000000000..fe32cead2039 --- /dev/null +++ b/docs/adding_timing_metadata.md @@ -0,0 +1,163 @@ +# Adding Timing Metadata + +## listing_meta.json files + +`listing_meta.json` files are SEMI AUTO-GENERATED. + +The raw data may be edited manually, to add entries or change timing values. + +The **list** of tests must stay up to date, so it can be used by external +tools. This is verified by presubmit checks. + +The `subcaseMS` values are estimates. They can be set to 0 if for some reason +you can't estimate the time (or there's an existing test with a long name and +slow subcases that would result in query strings that are too long), but this +will produce a non-fatal warning. Avoid creating new warnings whenever +possible. Any existing failures should be fixed (eventually). + +### Performance + +Note this data is typically captured by developers using higher-end +computers, so typical test machines might execute more slowly. For this +reason, the WPT chunking should be configured to generate chunks much shorter +than 5 seconds (a typical default time limit in WPT test executors) so they +should still execute in under 5 seconds on lower-end computers. + +## Problem + +When adding new tests to the CTS you may occasionally see an error like this +when running `npm test` or `npm run standalone`: + +``` +ERROR: Tests missing from listing_meta.json. Please add the new tests (set subcaseMS to 0 if you cannot estimate it): + webgpu:shader,execution,expression,binary,af_matrix_addition:matrix:* + +/home/runner/work/cts/cts/src/common/util/util.ts:38 + throw new Error(msg && (typeof msg === 'string' ? msg : msg())); + ^ +Error: + at assert (/home/runner/work/cts/cts/src/common/util/util.ts:38:11) + at crawl (/home/runner/work/cts/cts/src/common/tools/crawl.ts:155:11) +Warning: non-zero exit code 1 + Use --force to continue. + +Aborted due to warnings. +``` + +What this error message is trying to tell us, is that there is no entry for +`webgpu:shader,execution,expression,binary,af_matrix_addition:matrix:*` in +`src/webgpu/listing_meta.json`. + +These entries are estimates for the amount of time that subcases take to run, +and are used as inputs into the WPT tooling to attempt to portion out tests into +approximately same-sized chunks. + +If a value has been defaulted to 0 by someone, you will see warnings like this: + +``` +... +WARNING: subcaseMS≤0 found in listing_meta.json (allowed, but try to avoid): + webgpu:shader,execution,expression,binary,af_matrix_addition:matrix:* +... +``` + +These messages should be resolved by adding appropriate entries to the JSON +file. + +## Solution 1 (manual, best for simple tests) + +If you're developing new tests and need to update this file, it is sometimes +easiest to do so manually. Run your tests under your usual development workflow +and see how long they take. In the standalone web runner `npm start`, the total +time for a test case is reported on the right-hand side when the case logs are +expanded. + +Record the average time per *subcase* across all cases of the test (you may need +to compute this) into the `listing_meta.json` file. + +## Solution 2 (semi-automated) + +There exists tooling in the CTS repo for generating appropriate estimates for +these values, though they do require some manual intervention. The rest of this +doc will be a walkthrough of running these tools. + +Timing data can be captured in bulk and "merged" into this file using +the `merge_listing_times` tool. This is useful when a large number of tests +change or otherwise a lot of tests need to be updated, but it also automates the +manual steps above. + +The tool can also be used without any inputs to reformat `listing_meta.json`. +Please read the help message of `merge_listing_times` for more information. + +### Placeholder Value + +If your development workflow requires a clean build, the first step is to add a +placeholder value for entry to `src/webgpu/listing_meta.json`, since there is a +chicken-and-egg problem for updating these values. + +``` + "webgpu:shader,execution,expression,binary,af_matrix_addition:matrix:*": { "subcaseMS": 0 }, +``` + +(It should have a value of 0, since later tooling updates the value if the newer +value is higher.) + +### Websocket Logger + +The first tool that needs to be run is `websocket-logger`, which receives data +on a WebSocket channel to capture timing data when CTS is run. This +should be run in a separate process/terminal, since it needs to stay running +throughout the following steps. + +In the `tools/websocket-logger/` directory: + +``` +npm ci +npm start +``` + +The output from this command will indicate where the results are being logged, +which will be needed later. For example: + +``` +... +Writing to wslog-2023-09-12T18-57-34.txt +... +``` + +### Running CTS + +Now we need to run the specific cases in CTS that we need to time. +This should be possible under any development workflow (as long as its runtime environment, like Node, supports WebSockets), but the most well-tested way is using the standalone web runner. + +This requires serving the CTS locally. In the project root: + +``` +npm run standalone +npm start +``` + +Once this is started you can then direct a WebGPU enabled browser to the +specific CTS entry and run the tests, for example: + +``` +http://localhost:8080/standalone/?q=webgpu:shader,execution,expression,binary,af_matrix_addition:matrix:* +``` + +If the tests have a high variance in runtime, you can run them multiple times. +The longest recorded time will be used. + +### Merging metadata + +The final step is to merge the new data that has been captured into the JSON +file. + +This can be done using the following command: + +``` +tools/merge_listing_times webgpu -- tools/websocket-logger/wslog-2023-09-12T18-57-34.txt +``` + +where the text file is the result file from websocket-logger. + +Now you just need to commit the pending diff in your repo. diff --git a/docs/fp_primer.md b/docs/fp_primer.md index 6d0294b4d1e5..4d08d588f5e0 100644 --- a/docs/fp_primer.md +++ b/docs/fp_primer.md @@ -69,7 +69,7 @@ reference, see [binary64 on Wikipedia](https://en.wikipedia.org/wiki/Double-precision_floating-point_format), [binary32 on Wikipedia](https://en.wikipedia.org/wiki/Single-precision_floating-point_format), and -[binar16 on Wikipedia](https://en.wikipedia.org/wiki/Half-precision_floating-point_format). +[binary16 on Wikipedia](https://en.wikipedia.org/wiki/Half-precision_floating-point_format). In the floating points formats described above, there are two possible zero values, one with all bits being 0, called positive zero, and one all the same @@ -144,7 +144,7 @@ This concept of near-overflow vs far-overflow divides the real number line into | -∞ < `x` <= `-(2 ** (exp_max + 1))` | must round to -∞ | | `-(2 ** (exp_max + 1))` < `x` <= min fp value | must round to -∞ or min value | | min fp value < `x` < max fp value | round as discussed below | -| min fp value <= `x` < `2 ** (exp_max + 1)` | must round to max value or ∞ | +| max fp value <= `x` < `2 ** (exp_max + 1)` | must round to max value or ∞ | | `2 ** (exp_max + 1))` < `x` | implementations must round to ∞ | @@ -184,7 +184,7 @@ operations. Operations, which can be thought of as mathematical functions, are mappings from a set of inputs to a set of outputs. -Denoted `f(x, y) = X`, where f is a placeholder or the name of the operation, +Denoted `f(x, y) = X`, where `f` is a placeholder or the name of the operation, lower case variables are the inputs to the function, and uppercase variables are the outputs of the function. @@ -208,7 +208,7 @@ Some examples of different types of operations: `multiplication(x, y) = X`, which represents the WGSL expression `x * y`, takes in floating point values, `x` and `y`, and produces a floating point value `X`. -`lessThen(x, y) = X`, which represents the WGSL expression `x < y`, again takes +`lessThan(x, y) = X`, which represents the WGSL expression `x < y`, again takes in floating point values, but in this case returns a boolean value. `ldexp(x, y) = X`, which builds a floating point value, takes in a floating @@ -406,9 +406,9 @@ In more precise terms: X = [min(f(x)), max(f(x))] X = [min(f([a, b])), max(f([a, b]))] - X = [f(m), f(M)] + X = [f(m), f(n)] ``` -where m and M are in `[a, b]`, `m <= M`, and produce the min and max results +where `m` and `n` are in `[a, b]`, `m <= n`, and produce the min and max results for `f` on the interval, respectively. So how do we find the minima and maxima for our operation in the domain? @@ -499,15 +499,15 @@ literally pages of expanded intervals. sin(π/2) => [sin(π/2) - 2 ** -11, sin(π/2) + 2 ** -11] => [0 - 2 ** -11, 0 + 2 ** -11] - => [-0.000488.., 0.000488...] + => [-0.000488…, 0.000488…] cos(π/2) => [cos(π/2) - 2 ** -11, cos(π/2) + 2 ** -11] - => [-0.500488, -0.499511...] + => [-0.500488…, -0.499511…] tan(π/2) => sin(π/2)/cos(π/2) - => [-0.000488.., 0.000488...]/[-0.500488..., -0.499511...] - => [min({-0.000488.../-0.500488..., -0.000488.../-0.499511..., ...}), - max(min({-0.000488.../-0.500488..., -0.000488.../-0.499511..., ...}) ] - => [0.000488.../-0.499511..., 0.000488.../0.499511...] + => [-0.000488…, 0.000488…]/[-0.500488…, -0.499511…] + => [min(-0.000488…/-0.500488…, -0.000488…/-0.499511…, 0.000488…/-0.500488…, 0.000488…/-0.499511…), + max(-0.000488…/-0.500488…, -0.000488…/-0.499511…, 0.000488…/-0.500488…, 0.000488…/-0.499511…)] + => [0.000488…/-0.499511…, 0.000488…/0.499511…] => [-0.0009775171, 0.0009775171] ``` @@ -553,10 +553,10 @@ These are compile vs run time, and CPU vs GPU. Broadly speaking compile time execution happens on the host CPU, and run time evaluation occurs on a dedicated GPU. -(SwiftShader technically breaks this by being a software emulation of a GPU that -runs on the CPU, but conceptually one can think of SwiftShader has being a type -of GPU in this context, since it has similar constraints when it comes to -precision, etc.) +(Software graphics implementations like WARP and SwiftShader technically break this by +being a software emulation of a GPU that runs on the CPU, but conceptually one can +think of these implementations being a type of GPU in this context, since it has +similar constraints when it comes to precision, etc.) Compile time evaluation is execution that occurs when setting up a shader module, i.e. when compiling WGSL to a platform specific shading language. It is @@ -588,18 +588,18 @@ let c: f32 = a + b and ``` // compile time -const c: f32 = 1 + 2 +const c: f32 = 1.0f + 2.0f ``` -should produce the same result of `3` in the variable `c`, assuming `1` and `2` -were passed in as `a` & `b`. +should produce the same result of `3.0` in the variable `c`, assuming `1.0` and `2.0` +were passed in as `a` and `b`. The only difference, is when/where the execution occurs. The difference in behaviour between these two occur when the result of the operation is not finite for the underlying floating point type. -If instead of `1` and `2`, we had `10` and `f32.max`, so the true result is -`f32.max + 10`, the user will see a difference. Specifically the runtime +If instead of `1.0` and `2.0`, we had `10.0` and `f32.max`, so the true result is +`f32.max + 10.0`, the behaviours differ. Specifically the runtime evaluated version will still run, but the result in `c` will be an indeterminate value, which is any finite f32 value. For the compile time example instead, compiling the shader will fail validation. @@ -611,7 +611,7 @@ execution. Unfortunately we are dealing with intervals of results and not precise results. So this leads to more even conceptual complexity. For runtime evaluation, this -isn't too bad, because the rule becomes if any part of the interval is +isn't too bad, because the rule becomes: if any part of the interval is non-finite then an indeterminate value can be a result, and the interval for an indeterminate result `[fp min, fp max]`, will include any finite portions of the interval. diff --git a/package-lock.json b/package-lock.json index 9ff545a79a79..c4761e5f9aae 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23,7 +23,7 @@ "@types/pngjs": "^6.0.1", "@types/serve-index": "^1.9.1", "@typescript-eslint/parser": "^4.33.0", - "@webgpu/types": "gpuweb/types#ca1a548178567e6021fd194380b97be1bf6b07b7", + "@webgpu/types": "^0.1.38", "ansi-colors": "4.1.1", "babel-plugin-add-header-comment": "^1.0.3", "babel-plugin-const-enum": "^1.2.0", @@ -1262,11 +1262,10 @@ } }, "node_modules/@webgpu/types": { - "version": "0.1.34", - "resolved": "git+ssh://git@github.com/gpuweb/types.git#ca1a548178567e6021fd194380b97be1bf6b07b7", - "integrity": "sha512-L3q2iZPXqb5/qHupSV4G8tphM2GnCuaAf6SQWLqMNDgMwDk/Y4UWRxSIlY98ONKa1pDOuhIsXj6s1U3rU0wqhw==", - "dev": true, - "license": "BSD-3-Clause" + "version": "0.1.38", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.38.tgz", + "integrity": "sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA==", + "dev": true }, "node_modules/abbrev": { "version": "1.1.1", @@ -9884,10 +9883,10 @@ } }, "@webgpu/types": { - "version": "git+ssh://git@github.com/gpuweb/types.git#ca1a548178567e6021fd194380b97be1bf6b07b7", - "integrity": "sha512-L3q2iZPXqb5/qHupSV4G8tphM2GnCuaAf6SQWLqMNDgMwDk/Y4UWRxSIlY98ONKa1pDOuhIsXj6s1U3rU0wqhw==", - "dev": true, - "from": "@webgpu/types@gpuweb/types#ca1a548178567e6021fd194380b97be1bf6b07b7" + "version": "0.1.38", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.38.tgz", + "integrity": "sha512-7LrhVKz2PRh+DD7+S+PVaFd5HxaWQvoMqBbsV9fNJO1pjUs1P8bM2vQVNfk+3URTqbuTI7gkXi0rfsN0IadoBA==", + "dev": true }, "abbrev": { "version": "1.1.1", diff --git a/package.json b/package.json index 173a5094e63c..32c73858ef81 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "@types/pngjs": "^6.0.1", "@types/serve-index": "^1.9.1", "@typescript-eslint/parser": "^4.33.0", - "@webgpu/types": "gpuweb/types#ca1a548178567e6021fd194380b97be1bf6b07b7", + "@webgpu/types": "^0.1.38", "ansi-colors": "4.1.1", "babel-plugin-add-header-comment": "^1.0.3", "babel-plugin-const-enum": "^1.2.0", diff --git a/src/common/framework/data_cache.ts b/src/common/framework/data_cache.ts index 6f6e80288a8f..c1e3a889beb3 100644 --- a/src/common/framework/data_cache.ts +++ b/src/common/framework/data_cache.ts @@ -3,15 +3,64 @@ * expensive to build using a two-level cache (in-memory, pre-computed file). */ +import { assert } from '../util/util.js'; + interface DataStore { - load(path: string): Promise; + load(path: string): Promise; } /** Logger is a basic debug logger function */ export type Logger = (s: string) => void; -/** DataCache is an interface to a data store used to hold cached data */ +/** + * DataCacheNode represents a single cache entry in the LRU DataCache. + * DataCacheNode is a doubly linked list, so that least-recently-used entries can be removed, and + * cache hits can move the node to the front of the list. + */ +class DataCacheNode { + public constructor(path: string, data: unknown) { + this.path = path; + this.data = data; + } + + /** insertAfter() re-inserts this node in the doubly-linked list after `prev` */ + public insertAfter(prev: DataCacheNode) { + this.unlink(); + this.next = prev.next; + this.prev = prev; + prev.next = this; + if (this.next) { + this.next.prev = this; + } + } + + /** unlink() removes this node from the doubly-linked list */ + public unlink() { + const prev = this.prev; + const next = this.next; + if (prev) { + prev.next = next; + } + if (next) { + next.prev = prev; + } + this.prev = null; + this.next = null; + } + + public readonly path: string; // The file path this node represents + public readonly data: unknown; // The deserialized data for this node + public prev: DataCacheNode | null = null; // The previous node in the doubly-linked list + public next: DataCacheNode | null = null; // The next node in the doubly-linked list +} + +/** DataCache is an interface to a LRU-cached data store used to hold data cached by path */ export class DataCache { + public constructor() { + this.lruHeadNode.next = this.lruTailNode; + this.lruTailNode.prev = this.lruHeadNode; + } + /** setDataStore() sets the backing data store used by the data cache */ public setStore(dataStore: DataStore) { this.dataStore = dataStore; @@ -28,17 +77,20 @@ export class DataCache { * building the data and storing it in the cache. */ public async fetch(cacheable: Cacheable): Promise { - // First check the in-memory cache - let data = this.cache.get(cacheable.path); - if (data !== undefined) { - this.log('in-memory cache hit'); - return Promise.resolve(data as Data); + { + // First check the in-memory cache + const node = this.cache.get(cacheable.path); + if (node !== undefined) { + this.log('in-memory cache hit'); + node.insertAfter(this.lruHeadNode); + return Promise.resolve(node.data as Data); + } } this.log('in-memory cache miss'); // In in-memory cache miss. // Next, try the data store. if (this.dataStore !== null && !this.unavailableFiles.has(cacheable.path)) { - let serialized: string | undefined; + let serialized: Uint8Array | undefined; try { serialized = await this.dataStore.load(cacheable.path); this.log('loaded serialized'); @@ -49,16 +101,37 @@ export class DataCache { } if (serialized !== undefined) { this.log(`deserializing`); - data = cacheable.deserialize(serialized); - this.cache.set(cacheable.path, data); - return data as Data; + const data = cacheable.deserialize(serialized); + this.addToCache(cacheable.path, data); + return data; } } // Not found anywhere. Build the data, and cache for future lookup. this.log(`cache: building (${cacheable.path})`); - data = await cacheable.build(); - this.cache.set(cacheable.path, data); - return data as Data; + const data = await cacheable.build(); + this.addToCache(cacheable.path, data); + return data; + } + + /** + * addToCache() creates a new node for `path` and `data`, inserting the new node at the front of + * the doubly-linked list. If the number of entries in the cache exceeds this.maxCount, then the + * least recently used entry is evicted + * @param path the file path for the data + * @param data the deserialized data + */ + private addToCache(path: string, data: unknown) { + if (this.cache.size >= this.maxCount) { + const toEvict = this.lruTailNode.prev; + assert(toEvict !== null); + toEvict.unlink(); + this.cache.delete(toEvict.path); + this.log(`evicting ${toEvict.path}`); + } + const node = new DataCacheNode(path, data); + node.insertAfter(this.lruHeadNode); + this.cache.set(path, node); + this.log(`added ${path}. new count: ${this.cache.size}`); } private log(msg: string) { @@ -67,7 +140,12 @@ export class DataCache { } } - private cache = new Map(); + // Max number of entries in the cache before LRU entries are evicted. + private readonly maxCount = 4; + + private cache = new Map(); + private lruHeadNode = new DataCacheNode('', null); // placeholder node (no path or data) + private lruTailNode = new DataCacheNode('', null); // placeholder node (no path or data) private unavailableFiles = new Set(); private dataStore: DataStore | null = null; private debugLogger: Logger | null = null; @@ -107,14 +185,13 @@ export interface Cacheable { build(): Promise; /** - * serialize() transforms `data` to a string (usually JSON encoded) so that it - * can be stored in a text cache file. + * serialize() encodes `data` to a binary representation so that it can be stored in a cache file. */ - serialize(data: Data): string; + serialize(data: Data): Uint8Array; /** - * deserialize() is the inverse of serialize(), transforming the string back - * to the Data object. + * deserialize() is the inverse of serialize(), decoding the binary representation back to a Data + * object. */ - deserialize(serialized: string): Data; + deserialize(binary: Uint8Array): Data; } diff --git a/src/common/framework/fixture.ts b/src/common/framework/fixture.ts index 7722a4fe1b0c..795532406bd2 100644 --- a/src/common/framework/fixture.ts +++ b/src/common/framework/fixture.ts @@ -1,6 +1,6 @@ import { TestCaseRecorder } from '../internal/logging/test_case_recorder.js'; import { JSONWithUndefined } from '../internal/params_utils.js'; -import { assert, unreachable } from '../util/util.js'; +import { assert, ExceptionCheckOptions, unreachable } from '../util/util.js'; export class SkipTestCase extends Error {} export class UnexpectedPassError extends Error {} @@ -166,6 +166,13 @@ export class Fixture { throw new SkipTestCase(msg); } + /** Throws an exception marking the subcase as skipped if condition is true */ + skipIf(cond: boolean, msg: string | (() => string) = '') { + if (cond) { + this.skip(typeof msg === 'function' ? msg() : msg); + } + } + /** Log a warning and increase the result status to "Warn". */ warn(msg?: string): void { this.rec.warn(new Error(msg)); @@ -230,16 +237,26 @@ export class Fixture { } /** Expect that the provided promise rejects, with the provided exception name. */ - shouldReject(expectedName: string, p: Promise, msg?: string): void { + shouldReject( + expectedName: string, + p: Promise, + { allowMissingStack = false, message }: ExceptionCheckOptions = {} + ): void { this.eventualAsyncExpectation(async niceStack => { - const m = msg ? ': ' + msg : ''; + const m = message ? ': ' + message : ''; try { await p; niceStack.message = 'DID NOT REJECT' + m; this.rec.expectationFailed(niceStack); } catch (ex) { - niceStack.message = 'rejected as expected' + m; this.expectErrorValue(expectedName, ex, niceStack); + if (!allowMissingStack) { + if (!(ex instanceof Error && typeof ex.stack === 'string')) { + const exMessage = ex instanceof Error ? ex.message : '?'; + niceStack.message = `rejected as expected, but missing stack (${exMessage})${m}`; + this.rec.expectationFailed(niceStack); + } + } } }); } @@ -250,8 +267,12 @@ export class Fixture { * * MAINTENANCE_TODO: Change to `string | false` so the exception name is always checked. */ - shouldThrow(expectedError: string | boolean, fn: () => void, msg?: string): void { - const m = msg ? ': ' + msg : ''; + shouldThrow( + expectedError: string | boolean, + fn: () => void, + { allowMissingStack = false, message }: ExceptionCheckOptions = {} + ) { + const m = message ? ': ' + message : ''; try { fn(); if (expectedError === false) { @@ -264,6 +285,11 @@ export class Fixture { this.rec.expectationFailed(new Error('threw unexpectedly' + m)); } else { this.expectErrorValue(expectedError, ex, new Error(m)); + if (!allowMissingStack) { + if (!(ex instanceof Error && typeof ex.stack === 'string')) { + this.rec.expectationFailed(new Error('threw as expected, but missing stack' + m)); + } + } } } } diff --git a/src/common/framework/metadata.ts b/src/common/framework/metadata.ts new file mode 100644 index 000000000000..2c2a1ef79478 --- /dev/null +++ b/src/common/framework/metadata.ts @@ -0,0 +1,28 @@ +import { assert } from '../util/util.js'; + +/** Metadata about tests (that can't be derived at runtime). */ +export type TestMetadata = { + /** + * Estimated average time-per-subcase, in milliseconds. + * This is used to determine chunking granularity when exporting to WPT with + * chunking enabled (like out-wpt/cts-chunked2sec.https.html). + */ + subcaseMS: number; +}; + +export type TestMetadataListing = { + [testQuery: string]: TestMetadata; +}; + +export function loadMetadataForSuite(suiteDir: string): TestMetadataListing | null { + assert(typeof require !== 'undefined', 'loadMetadataForSuite is only implemented on Node'); + const fs = require('fs'); + + const metadataFile = `${suiteDir}/listing_meta.json`; + if (!fs.existsSync(metadataFile)) { + return null; + } + + const metadata: TestMetadataListing = JSON.parse(fs.readFileSync(metadataFile, 'utf8')); + return metadata; +} diff --git a/src/common/framework/params_builder.ts b/src/common/framework/params_builder.ts index 4947245a3251..845d1cd2e92a 100644 --- a/src/common/framework/params_builder.ts +++ b/src/common/framework/params_builder.ts @@ -1,6 +1,7 @@ import { Merged, mergeParams, mergeParamsChecked } from '../internal/params_utils.js'; import { comparePublicParamsPaths, Ordering } from '../internal/query/compare.js'; import { stringifyPublicParams } from '../internal/query/stringify_params.js'; +import { DeepReadonly } from '../util/types.js'; import { assert, mapLazy, objectEquals } from '../util/util.js'; import { TestParams } from './fixture.js'; @@ -98,7 +99,7 @@ export type ParamTypeOf< * - `[case params, undefined]` if not. */ export type CaseSubcaseIterable = Iterable< - readonly [CaseP, Iterable | undefined] + readonly [DeepReadonly, Iterable> | undefined] >; /** @@ -143,7 +144,7 @@ export function builderIterateCasesWithSubcases( */ export class CaseParamsBuilder extends ParamsBuilderBase - implements Iterable, ParamsBuilder { + implements Iterable>, ParamsBuilder { *iterateCasesWithSubcases(caseFilter: TestParams | null): CaseSubcaseIterable { for (const caseP of this.cases(caseFilter)) { if (caseFilter) { @@ -155,12 +156,12 @@ export class CaseParamsBuilder } } - yield [caseP, undefined]; + yield [caseP as DeepReadonly, undefined]; } } - [Symbol.iterator](): Iterator { - return this.cases(null); + [Symbol.iterator](): Iterator> { + return this.cases(null) as Iterator>; } /** @inheritDoc */ @@ -302,7 +303,10 @@ export class SubcaseParamsBuilder const subcases = Array.from(this.subcases(caseP)); if (subcases.length) { - yield [caseP, subcases]; + yield [ + caseP as DeepReadonly, + subcases as DeepReadonly[], + ]; } } } diff --git a/src/common/internal/file_loader.ts b/src/common/internal/file_loader.ts index 3b6afef7ac78..dddedf768830 100644 --- a/src/common/internal/file_loader.ts +++ b/src/common/internal/file_loader.ts @@ -69,16 +69,21 @@ export abstract class TestFileLoader extends EventTarget { return ret; } - async loadTree(query: TestQuery, subqueriesToExpand: string[] = []): Promise { - const tree = await loadTreeForQuery( - this, - query, - subqueriesToExpand.map(s => { + async loadTree( + query: TestQuery, + { + subqueriesToExpand = [], + maxChunkTime = Infinity, + }: { subqueriesToExpand?: string[]; maxChunkTime?: number } = {} + ): Promise { + const tree = await loadTreeForQuery(this, query, { + subqueriesToExpand: subqueriesToExpand.map(s => { const q = parseQuery(s); assert(q.level >= 2, () => `subqueriesToExpand entries should not be multi-file:\n ${q}`); return q; - }) - ); + }), + maxChunkTime, + }); this.dispatchEvent(new MessageEvent('finish')); return tree; } diff --git a/src/common/internal/logging/result.ts b/src/common/internal/logging/result.ts index 0de661b50ce9..3318e8c937d5 100644 --- a/src/common/internal/logging/result.ts +++ b/src/common/internal/logging/result.ts @@ -3,7 +3,7 @@ import { LogMessageWithStack } from './log_message.js'; // MAINTENANCE_TODO: Add warn expectations export type Expectation = 'pass' | 'skip' | 'fail'; -export type Status = 'running' | 'warn' | Expectation; +export type Status = 'notrun' | 'running' | 'warn' | Expectation; export interface TestCaseResult { status: Status; diff --git a/src/common/internal/logging/test_case_recorder.ts b/src/common/internal/logging/test_case_recorder.ts index 7507bbdec647..f5c3252b5c7d 100644 --- a/src/common/internal/logging/test_case_recorder.ts +++ b/src/common/internal/logging/test_case_recorder.ts @@ -3,26 +3,43 @@ import { globalTestConfig } from '../../framework/test_config.js'; import { now, assert } from '../../util/util.js'; import { LogMessageWithStack } from './log_message.js'; -import { Expectation, LiveTestCaseResult } from './result.js'; +import { Expectation, LiveTestCaseResult, Status } from './result.js'; enum LogSeverity { - Pass = 0, + NotRun = 0, Skip = 1, - Warn = 2, - ExpectFailed = 3, - ValidationFailed = 4, - ThrewException = 5, + Pass = 2, + Warn = 3, + ExpectFailed = 4, + ValidationFailed = 5, + ThrewException = 6, } const kMaxLogStacks = 2; const kMinSeverityForStack = LogSeverity.Warn; +function logSeverityToString(status: LogSeverity): Status { + switch (status) { + case LogSeverity.NotRun: + return 'notrun'; + case LogSeverity.Pass: + return 'pass'; + case LogSeverity.Skip: + return 'skip'; + case LogSeverity.Warn: + return 'warn'; + default: + return 'fail'; // Everything else is an error + } +} + /** Holds onto a LiveTestCaseResult owned by the Logger, and writes the results into it. */ export class TestCaseRecorder { - private result: LiveTestCaseResult; + readonly result: LiveTestCaseResult; + public nonskippedSubcaseCount: number = 0; private inSubCase: boolean = false; - private subCaseStatus = LogSeverity.Pass; - private finalCaseStatus = LogSeverity.Pass; + private subCaseStatus = LogSeverity.NotRun; + private finalCaseStatus = LogSeverity.NotRun; private hideStacksBelowSeverity = kMinSeverityForStack; private startTime = -1; private logs: LogMessageWithStack[] = []; @@ -42,31 +59,33 @@ export class TestCaseRecorder { } finish(): void { - assert(this.startTime >= 0, 'finish() before start()'); + // This is a framework error. If this assert is hit, it won't be localized + // to a test. The whole test run will fail out. + assert(this.startTime >= 0, 'internal error: finish() before start()'); const timeMilliseconds = now() - this.startTime; // Round to next microsecond to avoid storing useless .xxxx00000000000002 in results. this.result.timems = Math.ceil(timeMilliseconds * 1000) / 1000; + if (this.finalCaseStatus === LogSeverity.Skip && this.nonskippedSubcaseCount !== 0) { + this.threw(new Error('internal error: case is "skip" but has nonskipped subcases')); + } + // Convert numeric enum back to string (but expose 'exception' as 'fail') - this.result.status = - this.finalCaseStatus === LogSeverity.Pass - ? 'pass' - : this.finalCaseStatus === LogSeverity.Skip - ? 'skip' - : this.finalCaseStatus === LogSeverity.Warn - ? 'warn' - : 'fail'; // Everything else is an error + this.result.status = logSeverityToString(this.finalCaseStatus); this.result.logs = this.logs; } beginSubCase() { - this.subCaseStatus = LogSeverity.Pass; + this.subCaseStatus = LogSeverity.NotRun; this.inSubCase = true; } endSubCase(expectedStatus: Expectation) { + if (this.subCaseStatus !== LogSeverity.Skip) { + this.nonskippedSubcaseCount++; + } try { if (expectedStatus === 'fail') { if (this.subCaseStatus <= LogSeverity.Warn) { @@ -77,9 +96,7 @@ export class TestCaseRecorder { } } finally { this.inSubCase = false; - if (this.subCaseStatus > this.finalCaseStatus) { - this.finalCaseStatus = this.subCaseStatus; - } + this.finalCaseStatus = Math.max(this.finalCaseStatus, this.subCaseStatus); } } @@ -93,7 +110,8 @@ export class TestCaseRecorder { } info(ex: Error): void { - this.logImpl(LogSeverity.Pass, 'INFO', ex); + // We need this to use the lowest LogSeverity so it doesn't override the current severity for this test case. + this.logImpl(LogSeverity.NotRun, 'INFO', ex); } skipped(ex: SkipTestCase): void { @@ -112,6 +130,14 @@ export class TestCaseRecorder { this.logImpl(LogSeverity.ValidationFailed, 'VALIDATION FAILED', ex); } + passed(): void { + if (this.inSubCase) { + this.subCaseStatus = Math.max(this.subCaseStatus, LogSeverity.Pass); + } else { + this.finalCaseStatus = Math.max(this.finalCaseStatus, LogSeverity.Pass); + } + } + threw(ex: unknown): void { if (ex instanceof SkipTestCase) { this.skipped(ex); @@ -127,9 +153,9 @@ export class TestCaseRecorder { // Final case status should be the "worst" of all log entries. if (this.inSubCase) { - if (level > this.subCaseStatus) this.subCaseStatus = level; + this.subCaseStatus = Math.max(this.subCaseStatus, level); } else { - if (level > this.finalCaseStatus) this.finalCaseStatus = level; + this.finalCaseStatus = Math.max(this.finalCaseStatus, level); } // setFirstLineOnly for all logs except `kMaxLogStacks` stacks at the highest severity diff --git a/src/common/internal/query/compare.ts b/src/common/internal/query/compare.ts index e9f4b0150336..a9419b87c196 100644 --- a/src/common/internal/query/compare.ts +++ b/src/common/internal/query/compare.ts @@ -80,7 +80,8 @@ export function comparePublicParamsPaths(a: TestParams, b: TestParams): Ordering const commonKeys = new Set(aKeys.filter(k => k in b)); for (const k of commonKeys) { - if (!objectEquals(a[k], b[k])) { + // Treat +/-0.0 as different query by distinguishing them in objectEquals + if (!objectEquals(a[k], b[k], true)) { return Ordering.Unordered; } } diff --git a/src/common/internal/test_group.ts b/src/common/internal/test_group.ts index 76b110af68cd..6e13fbf47458 100644 --- a/src/common/internal/test_group.ts +++ b/src/common/internal/test_group.ts @@ -26,8 +26,11 @@ import { stringifyPublicParamsUniquely, } from '../internal/query/stringify_params.js'; import { validQueryPart } from '../internal/query/validQueryPart.js'; +import { DeepReadonly } from '../util/types.js'; import { assert, unreachable } from '../util/util.js'; +import { logToWebsocket } from './websocket_logger.js'; + export type RunFn = ( rec: TestCaseRecorder, expectations?: TestQueryWithExpectation[] @@ -41,6 +44,7 @@ export interface TestCaseID { export interface RunCase { readonly id: TestCaseID; readonly isUnimplemented: boolean; + computeSubcaseCount(): number; run( rec: TestCaseRecorder, selfQuery: TestQuerySingleCase, @@ -60,6 +64,8 @@ export function makeTestGroup(fixture: FixtureClass): Test export interface IterableTestGroup { iterate(): Iterable; validate(): void; + /** Returns the file-relative test paths of tests which have >0 cases. */ + collectNonEmptyTests(): { testPath: string[] }[]; } export interface IterableTest { testPath: string[]; @@ -74,9 +80,14 @@ export function makeTestGroupForUnitTesting( return new TestGroup(fixture); } -type TestFn = (t: F & { params: P }) => Promise | void; +/** Parameter name for batch number (see also TestBuilder.batch). */ +const kBatchParamName = 'batch__'; + +type TestFn = ( + t: F & { params: DeepReadonly

} +) => Promise | void; type BeforeAllSubcasesFn = ( - s: S & { params: P } + s: S & { params: DeepReadonly

} ) => Promise | void; export class TestGroup implements TestGroupBuilder { @@ -124,6 +135,16 @@ export class TestGroup implements TestGroupBuilder { test.validate(); } } + + collectNonEmptyTests(): { testPath: string[] }[] { + const testPaths = []; + for (const test of this.tests) { + if (test.computeCaseCount() > 0) { + testPaths.push({ testPath: test.testPath }); + } + } + return testPaths; + } } interface TestBuilderWithName extends TestBuilderWithParams { @@ -265,6 +286,7 @@ class TestBuilder { }; } + /** Perform various validation/"lint" chenks. */ validate(): void { const testPathString = this.testPath.join(kPathSeparator); assert(this.testFn !== undefined, () => { @@ -283,7 +305,7 @@ class TestBuilder { for (const [caseParams, subcases] of builderIterateCasesWithSubcases(this.testCases, null)) { for (const subcaseParams of subcases ?? [{}]) { const params = mergeParams(caseParams, subcaseParams); - assert(this.batchSize === 0 || !('batch__' in params)); + assert(this.batchSize === 0 || !(kBatchParamName in params)); // stringifyPublicParams also checks for invalid params values let testcaseString; @@ -304,6 +326,18 @@ class TestBuilder { } } + computeCaseCount(): number { + if (this.testCases === undefined) { + return 1; + } + + let caseCount = 0; + for (const [_caseParams, _subcases] of builderIterateCasesWithSubcases(this.testCases, null)) { + caseCount++; + } + return caseCount; + } + params( cases: ((unit: CaseParamsBuilder<{}>) => ParamsBuilderBase<{}, {}>) | ParamsBuilderBase<{}, {}> ): TestBuilder { @@ -348,24 +382,53 @@ class TestBuilder { *iterate(caseFilter: TestParams | null): IterableIterator { this.testCases ??= kUnitCaseParamsBuilder; + + // Remove the batch__ from the caseFilter because the params builder doesn't + // know about it (we don't add it until later in this function). + let filterToBatch: number | undefined; + const caseFilterWithoutBatch = caseFilter ? { ...caseFilter } : null; + if (caseFilterWithoutBatch && kBatchParamName in caseFilterWithoutBatch) { + const batchParam = caseFilterWithoutBatch[kBatchParamName]; + assert(typeof batchParam === 'number'); + filterToBatch = batchParam; + delete caseFilterWithoutBatch[kBatchParamName]; + } + for (const [caseParams, subcases] of builderIterateCasesWithSubcases( this.testCases, - caseFilter + caseFilterWithoutBatch )) { + // If batches are not used, yield just one case. if (this.batchSize === 0 || subcases === undefined) { yield this.makeCaseSpecific(caseParams, subcases); - } else { - const subcaseArray = Array.from(subcases); - if (subcaseArray.length <= this.batchSize) { - yield this.makeCaseSpecific(caseParams, subcaseArray); - } else { - for (let i = 0; i < subcaseArray.length; i = i + this.batchSize) { - yield this.makeCaseSpecific( - { ...caseParams, batch__: i / this.batchSize }, - subcaseArray.slice(i, Math.min(subcaseArray.length, i + this.batchSize)) - ); - } - } + continue; + } + + // Same if there ends up being only one batch. + const subcaseArray = Array.from(subcases); + if (subcaseArray.length <= this.batchSize) { + yield this.makeCaseSpecific(caseParams, subcaseArray); + continue; + } + + // There are multiple batches. Helper function for this case: + const makeCaseForBatch = (batch: number) => { + const sliceStart = batch * this.batchSize; + return this.makeCaseSpecific( + { ...caseParams, [kBatchParamName]: batch }, + subcaseArray.slice(sliceStart, Math.min(subcaseArray.length, sliceStart + this.batchSize)) + ); + }; + + // If we filter to just one batch, yield it. + if (filterToBatch !== undefined) { + yield makeCaseForBatch(filterToBatch); + continue; + } + + // Finally, if not, yield all of the batches. + for (let batch = 0; batch * this.batchSize < subcaseArray.length; ++batch) { + yield makeCaseForBatch(batch); } } } @@ -402,6 +465,18 @@ class RunCaseSpecific implements RunCase { this.testCreationStack = testCreationStack; } + computeSubcaseCount(): number { + if (this.subcases) { + let count = 0; + for (const _subcase of this.subcases) { + count++; + } + return count; + } else { + return 1; + } + } + async runTest( rec: TestCaseRecorder, sharedState: SubcaseBatchState, @@ -419,6 +494,7 @@ class RunCaseSpecific implements RunCase { try { await inst.init(); await this.fn(inst as Fixture & { params: {} }); + rec.passed(); } finally { // Runs as long as constructor succeeded, even if initialization or the test failed. await inst.finalize(); @@ -428,10 +504,10 @@ class RunCaseSpecific implements RunCase { // An error from init or test may have been a SkipTestCase. // An error from finalize may have been an eventualAsyncExpectation failure // or unexpected validation/OOM error from the GPUDevice. + rec.threw(ex); if (throwSkip && ex instanceof SkipTestCase) { throw ex; } - rec.threw(ex); } finally { try { rec.endSubCase(expectedStatus); @@ -624,6 +700,24 @@ class RunCaseSpecific implements RunCase { rec.threw(ex); } finally { rec.finish(); + + const msg: CaseTimingLogLine = { + q: selfQuery.toString(), + timems: rec.result.timems, + nonskippedSubcaseCount: rec.nonskippedSubcaseCount, + }; + logToWebsocket(JSON.stringify(msg)); } } } + +export type CaseTimingLogLine = { + q: string; + /** Total time it took to execute the case. */ + timems: number; + /** + * Number of subcases that ran in the case (excluding skipped subcases, so + * they don't dilute the average per-subcase time. + */ + nonskippedSubcaseCount: number; +}; diff --git a/src/common/internal/tree.ts b/src/common/internal/tree.ts index 812b88c59e91..594837059ca7 100644 --- a/src/common/internal/tree.ts +++ b/src/common/internal/tree.ts @@ -1,3 +1,4 @@ +import { loadMetadataForSuite, TestMetadataListing } from '../framework/metadata.js'; import { globalTestConfig } from '../framework/test_config.js'; import { RunCase, RunFn } from '../internal/test_group.js'; import { assert, now } from '../util/util.js'; @@ -48,12 +49,13 @@ interface TestTreeNodeBase { * one (e.g. s:f:* relative to s:f,*), but something that is readable. */ readonly readableRelativeName: string; - subtreeCounts?: { tests: number; nodesWithTODO: number }; + subtreeCounts?: { tests: number; nodesWithTODO: number; totalTimeMS: number }; + subcaseCount?: number; } export interface TestSubtree extends TestTreeNodeBase { readonly children: Map; - readonly collapsible: boolean; + collapsible: boolean; description?: string; readonly testCreationStack?: Error; } @@ -62,6 +64,7 @@ export interface TestTreeLeaf extends TestTreeNodeBase { readonly run: RunFn; readonly isUnimplemented?: boolean; subtreeCounts?: undefined; + subcaseCount: number; } export type TestTreeNode = TestSubtree | TestTreeLeaf; @@ -89,9 +92,8 @@ export class TestTree { readonly forQuery: TestQuery; readonly root: TestSubtree; - constructor(forQuery: TestQuery, root: TestSubtree) { + private constructor(forQuery: TestQuery, root: TestSubtree) { this.forQuery = forQuery; - TestTree.propagateCounts(root); this.root = root; assert( root.query.level === 1 && root.query.depthInLevel === 0, @@ -99,6 +101,24 @@ export class TestTree { ); } + static async create( + forQuery: TestQuery, + root: TestSubtree, + maxChunkTime: number + ): Promise { + const suite = forQuery.suite; + + let chunking = undefined; + if (Number.isFinite(maxChunkTime)) { + const metadata = loadMetadataForSuite(`./src/${suite}`); + assert(metadata !== null, `metadata for ${suite} is missing, but maxChunkTime was requested`); + chunking = { metadata, maxChunkTime }; + } + await TestTree.propagateCounts(root, chunking); + + return new TestTree(forQuery, root); + } + /** * Iterate through the leaves of a version of the tree which has been pruned to exclude * subtrees which: @@ -185,16 +205,51 @@ export class TestTree { } /** Propagate the subtreeTODOs/subtreeTests state upward from leaves to parent nodes. */ - static propagateCounts(subtree: TestSubtree): { tests: number; nodesWithTODO: number } { - subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 }; + static async propagateCounts( + subtree: TestSubtree, + chunking: { metadata: TestMetadataListing; maxChunkTime: number } | undefined + ): Promise<{ tests: number; nodesWithTODO: number; totalTimeMS: number; subcaseCount: number }> { + subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 }; + subtree.subcaseCount = 0; for (const [, child] of subtree.children) { if ('children' in child) { - const counts = TestTree.propagateCounts(child); + const counts = await TestTree.propagateCounts(child, chunking); subtree.subtreeCounts.tests += counts.tests; subtree.subtreeCounts.nodesWithTODO += counts.nodesWithTODO; + subtree.subtreeCounts.totalTimeMS += counts.totalTimeMS; + subtree.subcaseCount += counts.subcaseCount; + } else { + subtree.subcaseCount = child.subcaseCount; } } - return subtree.subtreeCounts; + + // If we're chunking based on a maxChunkTime, then at each + // TestQueryMultiCase node of the tree we look at its total time. If the + // total time is larger than the maxChunkTime, we set collapsible=false to + // make sure it gets split up in the output. Note: + // - TestQueryMultiTest and higher nodes are never set to collapsible anyway, so we ignore them. + // - TestQuerySingleCase nodes can't be collapsed, so we ignore them. + if (chunking && subtree.query instanceof TestQueryMultiCase) { + const testLevelQuery = new TestQueryMultiCase( + subtree.query.suite, + subtree.query.filePathParts, + subtree.query.testPathParts, + {} + ).toString(); + + const metadata = chunking.metadata; + + const subcaseTiming: number | undefined = metadata[testLevelQuery]?.subcaseMS; + if (subcaseTiming !== undefined) { + const totalTiming = subcaseTiming * subtree.subcaseCount; + subtree.subtreeCounts.totalTimeMS = totalTiming; + if (totalTiming > chunking.maxChunkTime) { + subtree.collapsible = false; + } + } + } + + return { ...subtree.subtreeCounts, subcaseCount: subtree.subcaseCount ?? 0 }; } /** Displays counts in the format `(Nodes with TODOs) / (Total test count)`. */ @@ -229,7 +284,10 @@ export class TestTree { export async function loadTreeForQuery( loader: TestFileLoader, queryToLoad: TestQuery, - subqueriesToExpand: TestQuery[] + { + subqueriesToExpand, + maxChunkTime = Infinity, + }: { subqueriesToExpand: TestQuery[]; maxChunkTime?: number } ): Promise { const suite = queryToLoad.suite; const specs = await loader.listing(suite); @@ -347,17 +405,17 @@ export async function loadTreeForQuery( isCollapsible ); // This is 1 test. Set tests=1 then count TODOs. - subtreeL2.subtreeCounts ??= { tests: 1, nodesWithTODO: 0 }; + subtreeL2.subtreeCounts ??= { tests: 1, nodesWithTODO: 0, totalTimeMS: 0 }; if (t.description) setSubtreeDescriptionAndCountTODOs(subtreeL2, t.description); - let paramsFilter = null; + let caseFilter = null; if ('params' in queryToLoad) { - paramsFilter = queryToLoad.params; + caseFilter = queryToLoad.params; } // MAINTENANCE_TODO: If tree generation gets too slow, avoid actually iterating the cases in a // file if there's no need to (based on the subqueriesToExpand). - for (const c of t.iterate(paramsFilter)) { + for (const c of t.iterate(caseFilter)) { // iterate() guarantees c's query is equal to or a subset of queryToLoad. if (queryToLoad instanceof TestQuerySingleCase) { @@ -391,7 +449,7 @@ export async function loadTreeForQuery( } assert(foundCase, `Query \`${queryToLoad.toString()}\` does not match any cases`); - return new TestTree(queryToLoad, subtreeL0); + return TestTree.create(queryToLoad, subtreeL0, maxChunkTime); } function setSubtreeDescriptionAndCountTODOs( @@ -400,7 +458,7 @@ function setSubtreeDescriptionAndCountTODOs( ) { assert(subtree.description === undefined); subtree.description = description.trim(); - subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0 }; + subtree.subtreeCounts ??= { tests: 0, nodesWithTODO: 0, totalTimeMS: 0 }; if (subtree.description.indexOf('TODO') !== -1) { subtree.subtreeCounts.nodesWithTODO++; } @@ -569,6 +627,7 @@ function insertLeaf(parent: TestSubtree, query: TestQuerySingleCase, t: RunCase) query, run: (rec, expectations) => t.run(rec, query, expectations || []), isUnimplemented: t.isUnimplemented, + subcaseCount: t.computeSubcaseCount(), }; // This is a leaf (e.g. s:f:t:x=1;* -> s:f:t:x=1). The key is always ''. diff --git a/src/common/internal/websocket_logger.ts b/src/common/internal/websocket_logger.ts new file mode 100644 index 000000000000..30246df843e4 --- /dev/null +++ b/src/common/internal/websocket_logger.ts @@ -0,0 +1,52 @@ +/** + * - 'uninitialized' means we haven't tried to connect yet + * - Promise means it's pending + * - 'failed' means it failed (this is the most common case, where the logger isn't running) + * - WebSocket means it succeeded + */ +let connection: Promise | WebSocket | 'failed' | 'uninitialized' = + 'uninitialized'; + +/** + * Log a string to a websocket at `localhost:59497`. See `tools/websocket-logger`. + * + * This does nothing if a connection couldn't be established on the first call. + */ +export function logToWebsocket(msg: string) { + if (connection === 'failed') { + return; + } + + if (connection === 'uninitialized') { + connection = new Promise(resolve => { + if (typeof WebSocket === 'undefined') { + resolve('failed'); + return; + } + + const ws = new WebSocket('ws://localhost:59497/optional_cts_websocket_logger'); + ws.onopen = () => { + resolve(ws); + }; + ws.onerror = () => { + connection = 'failed'; + resolve('failed'); + }; + ws.onclose = () => { + connection = 'failed'; + resolve('failed'); + }; + }); + void connection.then(resolved => { + connection = resolved; + }); + } + + void (async () => { + // connection may be a promise or a value here. Either is OK to await. + const ws = await connection; + if (ws !== 'failed') { + ws.send(msg); + } + })(); +} diff --git a/src/common/runtime/cmdline.ts b/src/common/runtime/cmdline.ts index 463546c06d32..44a73fb38b34 100644 --- a/src/common/runtime/cmdline.ts +++ b/src/common/runtime/cmdline.ts @@ -11,7 +11,7 @@ import { LiveTestCaseResult } from '../internal/logging/result.js'; import { parseQuery } from '../internal/query/parseQuery.js'; import { parseExpectationsForTestQuery } from '../internal/query/query.js'; import { Colors } from '../util/colors.js'; -import { setGPUProvider } from '../util/navigator_gpu.js'; +import { setDefaultRequestAdapterOptions, setGPUProvider } from '../util/navigator_gpu.js'; import { assert, unreachable } from '../util/util.js'; import sys from './helper/sys.js'; @@ -22,6 +22,7 @@ function usage(rc: number): never { tools/run_${sys.type} 'unittests:*' 'webgpu:buffers,*' Options: --colors Enable ANSI colors in output. + --compat Runs tests in compatibility mode. --coverage Emit coverage data. --verbose Print result/log of every test as it runs. --list Print all testcase names that match the given query and exit. @@ -99,6 +100,8 @@ for (let i = 0; i < sys.args.length; ++i) { quiet = true; } else if (a === '--unroll-const-eval-loops') { globalTestConfig.unrollConstEvalLoops = true; + } else if (a === '--compat') { + globalTestConfig.compatibility = true; } else { console.log('unrecognized flag: ', a); usage(1); @@ -110,6 +113,11 @@ for (let i = 0; i < sys.args.length; ++i) { let codeCoverage: CodeCoverageProvider | undefined = undefined; +if (globalTestConfig.compatibility) { + // MAINTENANCE_TODO: remove the cast once compatibilityMode is officially added + setDefaultRequestAdapterOptions({ compatibilityMode: true } as GPURequestAdapterOptions); +} + if (gpuProviderModule) { setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); if (emitCoverage) { @@ -127,8 +135,8 @@ Did you remember to build with code coverage instrumentation enabled?` if (dataPath !== undefined) { dataCache.setStore({ load: (path: string) => { - return new Promise((resolve, reject) => { - fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + return new Promise((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, (err, data) => { if (err !== null) { reject(err.message); } else { diff --git a/src/common/runtime/server.ts b/src/common/runtime/server.ts index d8caf001c022..8310784e3a2c 100644 --- a/src/common/runtime/server.ts +++ b/src/common/runtime/server.ts @@ -14,7 +14,7 @@ import { parseQuery } from '../internal/query/parseQuery.js'; import { TestQueryWithExpectation } from '../internal/query/query.js'; import { TestTreeLeaf } from '../internal/tree.js'; import { Colors } from '../util/colors.js'; -import { setGPUProvider } from '../util/navigator_gpu.js'; +import { setDefaultRequestAdapterOptions, setGPUProvider } from '../util/navigator_gpu.js'; import sys from './helper/sys.js'; @@ -23,6 +23,7 @@ function usage(rc: number): never { tools/run_${sys.type} [OPTIONS...] Options: --colors Enable ANSI colors in output. + --compat Run tests in compatibility mode. --coverage Add coverage data to each result. --data Path to the data cache directory. --verbose Print result/log of every test as it runs. @@ -84,6 +85,8 @@ for (let i = 0; i < sys.args.length; ++i) { if (a.startsWith('-')) { if (a === '--colors') { Colors.enabled = true; + } else if (a === '--compat') { + globalTestConfig.compatibility = true; } else if (a === '--coverage') { emitCoverage = true; } else if (a === '--data') { @@ -107,6 +110,11 @@ for (let i = 0; i < sys.args.length; ++i) { let codeCoverage: CodeCoverageProvider | undefined = undefined; +if (globalTestConfig.compatibility) { + // MAINTENANCE_TODO: remove the cast once compatibilityMode is officially added + setDefaultRequestAdapterOptions({ compatibilityMode: true } as GPURequestAdapterOptions); +} + if (gpuProviderModule) { setGPUProvider(() => gpuProviderModule!.create(gpuProviderFlags)); @@ -125,8 +133,8 @@ Did you remember to build with code coverage instrumentation enabled?` if (dataPath !== undefined) { dataCache.setStore({ load: (path: string) => { - return new Promise((resolve, reject) => { - fs.readFile(`${dataPath}/${path}`, 'utf8', (err, data) => { + return new Promise((resolve, reject) => { + fs.readFile(`${dataPath}/${path}`, (err, data) => { if (err !== null) { reject(err.message); } else { diff --git a/src/common/runtime/standalone.ts b/src/common/runtime/standalone.ts index da647de29a01..be5887c1721e 100644 --- a/src/common/runtime/standalone.ts +++ b/src/common/runtime/standalone.ts @@ -11,7 +11,7 @@ import { parseQuery } from '../internal/query/parseQuery.js'; import { TestQueryLevel } from '../internal/query/query.js'; import { TestTreeNode, TestSubtree, TestTreeLeaf, TestTree } from '../internal/tree.js'; import { setDefaultRequestAdapterOptions } from '../util/navigator_gpu.js'; -import { assert, ErrorWithExtra, unreachable } from '../util/util.js'; +import { ErrorWithExtra, unreachable } from '../util/util.js'; import { kCTSOptionsInfo, @@ -84,7 +84,7 @@ dataCache.setStore({ if (!response.ok) { return Promise.reject(response.statusText); } - return await response.text(); + return new Uint8Array(await response.arrayBuffer()); }, }); @@ -321,7 +321,7 @@ function makeSubtreeHTML(n: TestSubtree, parentLevel: TestQueryLevel): Visualize if (subtreeResult.fail > 0) { status += 'fail'; } - if (subtreeResult.skip === subtreeResult.total) { + if (subtreeResult.skip === subtreeResult.total && subtreeResult.total > 0) { status += 'skip'; } div.setAttribute('data-status', status); @@ -390,6 +390,19 @@ function makeTreeNodeHeaderHTML( const div = $('

').addClass('nodeheader'); const header = $('').appendTo(div); + // prevent toggling if user is selecting text from an input element + { + let lastNodeName = ''; + div.on('pointerdown', event => { + lastNodeName = event.target.nodeName; + }); + div.on('click', event => { + if (lastNodeName === 'INPUT') { + event.preventDefault(); + } + }); + } + const setChecked = () => { div.prop('open', true); // (does not fire onChange) onChange(true); @@ -428,6 +441,14 @@ function makeTreeNodeHeaderHTML( .attr('alt', kOpenTestLinkAltText) .attr('title', kOpenTestLinkAltText) .appendTo(header); + $('