diff --git a/tests/helper.mjs b/tests/helper.mjs index 56d3724ad..455840e0c 100644 --- a/tests/helper.mjs +++ b/tests/helper.mjs @@ -4,9 +4,12 @@ import serve from "./server.mjs"; import { Builder, Capabilities, logging } from "selenium-webdriver"; +export const DEFAULT_RETRIES = 1; + const optionDefinitions = [ { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." }, { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." }, + { name: "retry", type: Number, defaultValue: DEFAULT_RETRIES, description: "Number of retries for the tests on failure." }, { name: "help", alias: "h", description: "Print this help text." }, ]; @@ -38,6 +41,9 @@ export default async function testSetup(helpText) { if (!BROWSER) printHelp("No browser specified, use $BROWSER or --browser", 1); + if (options.retry < 0) + printHelp("Number of retries cannot be negative", 1); + let capabilities; switch (BROWSER) { case "safari": @@ -86,5 +92,5 @@ export default async function testSetup(helpText) { if (driver) driver.close(); } - return { driver, PORT, stop }; + return { driver, PORT, stop, retry: options.retry }; } diff --git a/tests/run-end2end.mjs b/tests/run-end2end.mjs index 067359694..cdb81ebb4 100644 --- a/tests/run-end2end.mjs +++ b/tests/run-end2end.mjs @@ -1,7 +1,7 @@ #! /usr/bin/env node import assert from "assert"; -import testSetup from "./helper.mjs"; +import testSetup, { DEFAULT_RETRIES } from "./helper.mjs"; import { benchmarkConfigurator } from "../resources/benchmark-configurator.mjs"; const HELP = ` @@ -9,7 +9,7 @@ This script runs end2end tests by invoking the benchmark via the main Speedometer page in /index.html. `.trim(); -const { driver, PORT, stop } = await testSetup(HELP); +const { driver, PORT, stop, retry } = await testSetup(HELP); const suites = benchmarkConfigurator.suites; @@ -112,17 +112,36 @@ async function testDeveloperMode() { async function test() { try { - await driver.manage().setTimeouts({ script: 60000 }); - await testIterations(); - await testAll(); - await testDeveloperMode(); - console.log("\nTests complete!"); - } catch (e) { - console.error("\nTests failed!"); - throw e; + await testWithRetries(retry); } finally { stop(); } } +const RETRY_DELAY_MS = 3000; +async function testWithRetries(retries = DEFAULT_RETRIES) { + const maxAttempts = retries + 1; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + await tryTests(); + return; + } catch (e) { + console.error(`\nTests failed on attempt ${attempt}!`); + if (attempt === maxAttempts) + throw e; + + console.log(`Retrying... (${attempt}/${maxAttempts})`); + await new Promise((resolve) => setTimeout(resolve, RETRY_DELAY_MS)); + } + } +} + +async function tryTests() { + await driver.manage().setTimeouts({ script: 60000 }); + await testIterations(); + await testAll(); + await testDeveloperMode(); + console.log("\nTests complete!"); +} + setImmediate(test);