Skip to content

Commit

Permalink
Add a mechanism for expected failures in test262 tests
Browse files Browse the repository at this point in the history
This was present in the original Python script but not ported as we didn't
have any expected failures at the time.
  • Loading branch information
ptomato committed Jul 2, 2021
1 parent c6bbb01 commit 51310e4
Showing 1 changed file with 38 additions and 10 deletions.
48 changes: 38 additions & 10 deletions polyfill/test/parseResults.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,25 @@
#!/usr/bin/env node

const { stdin, stdout, exit } = process;
const fs = require('fs');

const PREFIXES = [
['FAIL', 'PASS'],
['EXPECTED FAIL', 'UNEXPECTED PASS']
];

let testOutput = '';
const now = Date.now();

const expectedFailures = new Set();
const lines = fs.readFileSync('expected-failures.txt', { encoding: 'utf-8' });
for (let line of lines.split('\n')) {
line = line.trim();
if (!line) continue;
if (line.startsWith('#')) continue;
expectedFailures.add(line);
}

stdin.setEncoding('utf8');

stdout.write('Starting Test262 tests.\n');
Expand Down Expand Up @@ -33,21 +49,33 @@ stdin.on('end', function () {
// filter out the non-strict tests because they were skipped via a preprocessor
const tests = JSON.parse(testOutput).filter((test) => test.scenario.includes('strict'));
const failedTests = [];
const unexpectedPasses = [];
for (const test of tests) {
const { result, scenario, file } = test;
const message = `${result.pass ? 'PASS' : 'FAIL'} ${file} (${scenario})\n`;
const expectedFailure = expectedFailures.has(file);
const message = `${PREFIXES[+expectedFailure][+result.pass]} ${file} (${scenario})\n`;
stdout.write(message);
if (!result.pass) failedTests.push({ ...test, message });
if (result.pass === expectedFailure) {
(result.pass ? unexpectedPasses : failedTests).push({ ...test, message });
}
}

if (failedTests.length) {
stdout.write(`\n${failedTests.length} tests failed:\n`);
for (const test of failedTests) {
const { message, rawResult } = test;
stdout.write(`\n${message}\n`);
stdout.write(`${rawResult.stderr}\n`);
stdout.write(`${rawResult.stdout}\n`);
stdout.write(rawResult.message ? rawResult.message + '\n' : '');
if (failedTests.length || unexpectedPasses.length) {
if (failedTests.length) {
stdout.write(`\n${failedTests.length} tests failed:\n`);
for (const test of failedTests) {
const { message, rawResult } = test;
stdout.write(`\n${message}\n`);
stdout.write(`${rawResult.stderr}\n`);
stdout.write(`${rawResult.stdout}\n`);
stdout.write(rawResult.message ? rawResult.message + '\n' : '');
}
}
if (unexpectedPasses.length) {
stdout.write(`\n${unexpectedPasses.length} tests passed unexpectedly:\n`);
for (const { message } of unexpectedPasses) {
stdout.write(`${message}\n`);
}
}
} else {
stdout.write('All results as expected.\n');
Expand Down

0 comments on commit 51310e4

Please sign in to comment.