Skip to content

Commit

Permalink
Revert "wip"
Browse files Browse the repository at this point in the history 
This reverts commit 38cafbf.
  • Loading branch information
@peaBerberian
peaBerberian committed Jan 20, 2025
1 parent d062755 commit a090fa5
Showing 1 changed file with 58 additions and 59 deletions.
117 changes: 58 additions & 59 deletions tests/performance/run.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ const PERF_TESTS_PORT = 8080;
*
* TODO: GitHub actions fails when running the 128th browser. Find out why.
*/
const TEST_ITERATIONS = 50;
const TEST_ITERATIONS = 30;

/**
* After initialization is done, contains the path allowing to run the Chrome
Expand Down Expand Up @@ -82,12 +82,6 @@ let currentBrowser;
*/
const tasks = [];

/**
* Index of the currently ran task in the `tasks` array.
* @see tasks
*/
let nextTaskIndex = 0;

/**
* Store results of the performance tests in two arrays:
* - "current" contains the test results of the current RxPlayer version
Expand All @@ -98,12 +92,6 @@ const allSamples = {
previous: [],
};

/**
* Contains references to every launched servers, with a `close` method allowing
* to close each one of them.
*/
const servers = [];

// If true, this script is called directly
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
const args = process.argv.slice(2);
Expand Down Expand Up @@ -173,7 +161,11 @@ if (import.meta.url === pathToFileURL(process.argv[1]).href) {
}
}

startPerformanceTests({ branchName, remote })
initializePerformanceTestsPages({
branchName: branchName ?? "dev",
remoteGitUrl: remote,
})
.then(() => runPerformanceTests())
.then(async (results) => {
const shouldWriteReportFile =
reportFile !== undefined &&
Expand Down Expand Up @@ -217,7 +209,7 @@ if (import.meta.url === pathToFileURL(process.argv[1]).href) {
}
console.warn("\nRetrying one time just to check if unlucky...");

const results2 = await startPerformanceTests({ branchName, remote });
const results2 = await runPerformanceTests();
appendToReportFile(
"\nPerformance tests 2nd run output:\n" + "---------------------------------",
);
Expand Down Expand Up @@ -302,39 +294,54 @@ function formatResultInHumanReadableWay(results, indent = 0) {

/**
* Initialize and start all tests on Chrome.
* @param {Object} opts - Various options to configure performance tests.
* @param {string} [opts.branchName] - The name of the branch results should be
* compared to.
* `dev` by default.
* @param {string} [opts.remoteGitUrl] - The git URL where the current
* repository can be cloned for comparisons.
* The one for the current git repository by default.
* @returns {Promise.<Object>}
*/
function startPerformanceTests({ branchName, remoteGitUrl } = {}) {
function runPerformanceTests() {
return new Promise((resolve, reject) => {
nextTaskIndex = 0;
let isFinished = false;
let contentServer;
let resultServer;
let staticServer;

const onFinished = () => {
isFinished = true;
closeServers();
const results = compareSamples();
shutdown().catch((err) => {
closeBrowser().catch((err) => {
// eslint-disable-next-line no-console
console.error("Failed to shutdown:", err);
console.error("Failed to close the browser:", err);
});
resolve(results);
};
const onError = (error) => {
shutdown().catch((err) => {
isFinished = true;
closeServers();
closeBrowser().catch((err) => {
// eslint-disable-next-line no-console
console.error("Failed to shutdown:", err);
console.error("Failed to close the browser:", err);
});
reject(error);
};
initScripts({
branchName: branchName ?? "dev",
remoteGitUrl,
})
.then(() => initServers(onFinished, onError))
.then(startAllTestsOnChrome)

const closeServers = () => {
contentServer?.close();
contentServer = undefined;
resultServer?.close();
resultServer = undefined;
staticServer?.close();
resultServer = undefined;
};

initServers(onFinished, onError)
.then((servers) => {
contentServer = servers.contentServer;
resultServer = servers.resultServer;
staticServer = servers.staticServer;
if (isFinished) {
closeServers();
}
return startAllTestsOnChrome();
})
.catch(onError);
});
}
Expand Down Expand Up @@ -379,7 +386,7 @@ async function initServers(onFinished, onError) {
* The one for the current git repository by default.
* @returns {Promise} - Resolves when the initialization is finished.
*/
async function initScripts({ branchName, remoteGitUrl }) {
async function initializePerformanceTestsPages({ branchName, remoteGitUrl }) {
await prepareLastRxPlayerTests({ branchName, remoteGitUrl });
await prepareCurrentRxPlayerTests();
}
Expand Down Expand Up @@ -467,12 +474,6 @@ async function linkRxPlayerBranch({ branchName, remoteGitUrl }) {
[],
(code) => new Error(`npm run build exited with code ${code}`),
).promise;
await spawnProc(
`cd ${rxPlayerPath} && cat VERSION`,

[],
(code) => new Error(`npm run build exited with code ${code}`),
).promise;

// GitHub actions, for unknown reasons, want to use the root's `dist` directory
// TODO: find why
Expand All @@ -488,6 +489,7 @@ async function linkRxPlayerBranch({ branchName, remoteGitUrl }) {
*/
async function startAllTestsOnChrome() {
CHROME_CMD = await getChromeCmd();
tasks.length = 0;
for (let i = 0; i < TEST_ITERATIONS; i++) {
tasks.push(() => startTestsOnChrome(i % 2 === 0, i + 1, TEST_ITERATIONS));
}
Expand All @@ -497,21 +499,17 @@ async function startAllTestsOnChrome() {
if (tasks.length === 0) {
throw new Error("No task scheduled");
}
nextTaskIndex++;
return tasks[nextTaskIndex - 1]();
return tasks.shift()();
}

/**
* Free all resources and terminate script.
*/
async function shutdown() {
async function closeBrowser() {
if (currentBrowser !== undefined) {
currentBrowser.kill("SIGKILL");
currentBrowser = undefined;
}
while (servers.length > 0) {
servers.pop().close();
}
}

/**
Expand All @@ -520,11 +518,11 @@ async function shutdown() {
* @param {Function} onFinished
*/
function startNextTaskOrFinish(onFinished) {
if (tasks[nextTaskIndex] === undefined) {
const nextTask = tasks.shift();
if (nextTask === undefined) {
onFinished();
}
nextTaskIndex++;
return tasks[nextTaskIndex - 1]();
return nextTask();
}

/**
Expand Down Expand Up @@ -730,8 +728,9 @@ function compareSamples() {
const resultCurrent = getResultsForSample(sampleCurrent);
const resultPrevious = getResultsForSample(samplePrevious);

const differenceMs = resultPrevious.mean - resultCurrent.mean;
const differencePc = differenceMs / resultCurrent.mean;
const medianDiffMs = resultPrevious.median - resultCurrent.median;
const meanDiffMs = resultPrevious.mean - resultCurrent.mean;
const differencePc = meanDiffMs / resultCurrent.mean;
const uValue = getUValueFromSamples(sampleCurrent, samplePrevious);
const zScore = Math.abs(
calculateZScore(uValue, sampleCurrent.length, samplePrevious.length),
Expand Down Expand Up @@ -763,28 +762,28 @@ function compareSamples() {
console.log("");
console.log(" Results");
console.log(` mean difference % (negative is slower): ${differencePc * 100}%`);
console.log(` mean difference time (negative is slower): ${differenceMs} ms`);
console.log(` mean difference time (negative is slower): ${meanDiffMs} ms`);
if (isSignificant) {
console.log(` The difference is significant (z: ${zScore})`);
if (differenceMs < -2) {
if (meanDiffMs < -2 && medianDiffMs < -2) {
results.worse.push({
testName,
previousMean: resultPrevious.mean,
currentMean: resultCurrent.mean,
previousMedian: resultPrevious.median,
currentMedian: resultCurrent.median,
differenceMs,
differenceMs: meanDiffMs,
differencePc,
zScore,
});
} else if (differenceMs > 2) {
} else if (meanDiffMs > 2 && medianDiffMs > 2) {
results.better.push({
testName,
previousMean: resultPrevious.mean,
currentMean: resultCurrent.mean,
previousMedian: resultPrevious.median,
currentMedian: resultCurrent.median,
differenceMs,
differenceMs: meanDiffMs,
differencePc,
zScore,
});
Expand All @@ -795,7 +794,7 @@ function compareSamples() {
currentMean: resultCurrent.mean,
previousMedian: resultPrevious.median,
currentMedian: resultCurrent.median,
differenceMs,
differenceMs: meanDiffMs,
differencePc,
zScore,
});
Expand All @@ -808,7 +807,7 @@ function compareSamples() {
currentMean: resultCurrent.mean,
previousMedian: resultPrevious.median,
currentMedian: resultCurrent.median,
differenceMs,
differenceMs: meanDiffMs,
differencePc,
zScore,
});
Expand Down

0 comments on commit a090fa5

Please sign in to comment.