diff --git a/Helpers/BrowserDrivers.cs b/Helpers/BrowserDrivers.cs index 481fd59..f162ed8 100644 --- a/Helpers/BrowserDrivers.cs +++ b/Helpers/BrowserDrivers.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using OpenQA.Selenium; @@ -65,6 +66,22 @@ public static class BrowserDrivers /// public const string EdgeDriverVariable = "EDGEWEBDRIVER"; + /// + /// Seconds the first start of a browser in a run may take. A + /// browser's first start on a fresh runner can take minutes, where + /// the driver allows each command 60 seconds, so the first start is + /// made by with this limit instead. + /// + public const int FirstStartTimeoutSeconds = 300; + + /// Page the first start of a browser loads. + private const string FirstStartPage = "about:blank"; + + private static readonly object _firstStartLock = new object(); + + private static readonly Dictionary _firstStarts = + new Dictionary(); + /// Path to the Chrome or Chromium binary to drive. public const string ChromeBinaryVariable = "CHROME_BIN"; @@ -125,6 +142,12 @@ public static WebDriver CreateChrome(ChromeOptions options) } SetBinaryLocation(options, ChromeBinaryVariable, _chromeBinaryNames); var driverPath = FindDriver(ChromeDriverVariable, ChromeDriverNames); + StartOnce("Chrome", timeout => Visit(new ChromeDriver( + driverPath != null + ? ChromeDriverService.CreateDefaultService(driverPath) + : ChromeDriverService.CreateDefaultService(), + options, + timeout))); if (driverPath != null) { return new ChromeDriver( @@ -154,6 +177,12 @@ public static WebDriver CreateFirefox(FirefoxOptions options) SetBinaryLocation( options, FirefoxBinaryVariable, _firefoxBinaryNames); var driverPath = FindDriver(GeckoDriverVariable, GeckoDriverNames); + StartOnce("Firefox", timeout => Visit(new FirefoxDriver( + driverPath != null + ? FirefoxDriverService.CreateDefaultService(driverPath) + : FirefoxDriverService.CreateDefaultService(), + options, + timeout))); if (driverPath != null) { return new FirefoxDriver( @@ -166,7 +195,10 @@ public static WebDriver CreateFirefox(FirefoxOptions options) } /// - /// Creates an Edge driver. + /// Creates an Edge driver. Edge is not started once first, as Chrome + /// and Firefox are, because headless Edge on the Ubuntu runners hangs + /// on its first page, which would hold the run for the whole of + /// . /// /// Options for the browser. /// A driver, which the caller quits. @@ -201,6 +233,102 @@ public static WebDriver CreateEdge(EdgeOptions options) "Edge", EdgeDriverVariable, EdgeDriverNames); } + /// + /// The line written at the first start of each browser in this run, + /// by the browser's name. + /// + public static IReadOnlyDictionary FirstStarts + { + get + { + lock (_firstStartLock) + { + return new Dictionary(_firstStarts); + } + } + } + + /// + /// Runs the first time a browser is asked + /// for in this run, and never again, so that the browser's slow first + /// start is made before the first test that uses it and not inside + /// one of that test's commands. Writes how long it took to the output + /// of the test that asked. Whatever stops the start is written and + /// not thrown, and the test then meets it when it starts the browser + /// itself. + /// + /// Name of the browser. + /// + /// Starts the browser, loads a page and quits, with the time limit + /// for each command it is given. + /// + public static void StartOnce(string browser, Action start) + { + if (browser == null) + { + throw new ArgumentNullException(nameof(browser)); + } + if (start == null) + { + throw new ArgumentNullException(nameof(start)); + } + lock (_firstStartLock) + { + if (_firstStarts.ContainsKey(browser)) + { + return; + } + var watch = Stopwatch.StartNew(); + string line; + try + { + start(TimeSpan.FromSeconds(FirstStartTimeoutSeconds)); + line = $"Started {browser} once before its first test " + + "in " + Seconds(watch) + " s."; + } + catch (Exception e) + { + line = $"{browser} did not finish its first start in " + + Seconds(watch) + " s, so the test starts it as it " + + "would have. " + FirstLine(e.Message); + } + _firstStarts[browser] = line; + Console.WriteLine(line); + } + } + + /// + /// Loads a page in and quits it. + /// + /// A driver just created. + private static void Visit(WebDriver driver) + { + try + { + driver.Navigate().GoToUrl(FirstStartPage); + } + finally + { + driver.Quit(); + } + } + + /// Seconds on the watch, to one decimal place. + /// A running watch. + /// The seconds, written the same in every culture. + private static string Seconds(Stopwatch watch) => + watch.Elapsed.TotalSeconds.ToString( + "0.0", System.Globalization.CultureInfo.InvariantCulture); + + /// + /// The first line of , which for a driver's + /// message is the reason without the details that follow it. + /// + /// Text of one or more lines. + /// The first line, or an empty string. + private static string FirstLine(string text) => + (text ?? string.Empty).Split('\n')[0].TrimEnd('\r'); + /// /// Runs , which leaves Selenium Manager to /// fetch a driver, and on failure says what was looked for here first, diff --git a/README.md b/README.md index 5bdbf4a..ae48ad5 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,11 @@ Select a subset with `--filter TestCategory=Contract` or - **Browser** - a driver the machine already provides is used first, named by `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path. When the machine provides none, Selenium Manager fetches one. Set - `SELENIUM_URL` to drive a browser on a Selenium grid instead. + `SELENIUM_URL` to drive a browser on a Selenium grid instead. Chrome and + Firefox are each started once, before the first test that uses them, with + up to 300 seconds for that first start, because a browser's first start on a + fresh runner can take longer than the 60 seconds a test's command is given. + The time it took is written to that test's output. ## Configuration diff --git a/Selenium/FirstStartTests.cs b/Selenium/FirstStartTests.cs new file mode 100644 index 0000000..3abbfb9 --- /dev/null +++ b/Selenium/FirstStartTests.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; +using System.Threading; +using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers; +using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenQA.Selenium; +using OpenQA.Selenium.Chrome; +using OpenQA.Selenium.Firefox; + +namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium +{ + /// + /// Checks that each browser is started once before the first test that + /// uses it, so that a slow first start is not spent inside a test. + /// + /// + /// is reached by name, so that a build + /// without the start once fails these tests rather than the build. + /// + [TestClass] + public class FirstStartTests + { + private const string StartOnceName = "StartOnce"; + private const string FirstStartsName = "FirstStarts"; + private const string TimeoutName = "FirstStartTimeoutSeconds"; + + /// + /// A browser's start is run once, however many times it is asked + /// for, and is given the first start's time limit. + /// + [TestMethod] + public void StartOnce_StartsEachBrowserOnce() + { + var browser = "Stand-in " + Guid.NewGuid(); + var starts = 0; + var limit = TimeSpan.Zero; + Action start = timeout => + { + starts++; + limit = timeout; + }; + + StartOnce(browser, start); + StartOnce(browser, start); + + Assert.AreEqual(1, starts); + Assert.AreEqual( + TimeSpan.FromSeconds(TimeoutSeconds()), + limit); + StringAssert.StartsWith( + FirstStarts()[browser], + $"Started {browser} once before its first test in "); + } + + /// + /// A first start that fails is reported and not thrown, and is not + /// tried again, so the test that follows meets the failure itself. + /// + [TestMethod] + public void StartOnce_AStartThatFails_IsReportedAndNotThrown() + { + var browser = "Stand-in " + Guid.NewGuid(); + var starts = 0; + Action start = timeout => + { + starts++; + throw new WebDriverException( + "session not created\nsecond line"); + }; + + StartOnce(browser, start); + StartOnce(browser, start); + + Assert.AreEqual(1, starts); + var line = FirstStarts()[browser]; + StringAssert.StartsWith( + line, $"{browser} did not finish its first start in "); + StringAssert.EndsWith(line, "session not created"); + } + + /// + /// Creating a Chrome driver starts Chrome once first. + /// + [TestMethod, TestCategory("Browser")] + public void Chrome_StartsOnceBeforeItsFirstTest() + { + var options = new ChromeOptions(); + options.AddArgument("--headless"); + options.AddArgument("--no-sandbox"); + options.AddArgument("--disable-dev-shm-usage"); + AssertStartedOnce( + "Chrome", () => BrowserDrivers.CreateChrome(options)); + } + + /// + /// Creating a Firefox driver starts Firefox once first. + /// + [TestMethod, TestCategory("Browser")] + public void Firefox_StartsOnceBeforeItsFirstTest() + { + var options = new FirefoxOptions(); + options.AddArgument("--headless"); + AssertStartedOnce( + "Firefox", () => BrowserDrivers.CreateFirefox(options)); + } + + /// + /// Creates the browser twice, loading a page each time, writes how + /// long each took, and checks the first start was made and kept. + /// + /// Name the first start is kept under. + /// Creates the driver. + private static void AssertStartedOnce( + string browser, Func create) + { + var url = + $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/"; + using var cts = new CancellationTokenSource(); + var server = TestHelpers.SimpleListener( + url, "First start", cts.Token); + try + { + var times = new List(); + for (var i = 0; i < 2; i++) + { + var watch = Stopwatch.StartNew(); + var driver = create(); + try + { + driver.Navigate().GoToUrl(url); + } + finally + { + driver.Quit(); + } + times.Add(watch.Elapsed.TotalSeconds); + } + var starts = FirstStarts(); + Assert.IsTrue( + starts.ContainsKey(browser), + $"{browser} was not started once before its first test."); + Console.WriteLine(starts[browser]); + Console.WriteLine(FormattableString.Invariant( + $"{browser} then started and loaded a page in ") + + FormattableString.Invariant( + $"{times[0]:0.0} s and {times[1]:0.0} s.")); + } + finally + { + cts.Cancel(); + server?.Listener?.Stop(); + server?.Listener?.Close(); + } + } + + private static void StartOnce(string browser, Action start) + { + var method = typeof(BrowserDrivers).GetMethod( + StartOnceName, + BindingFlags.Public | BindingFlags.Static, + new[] { typeof(string), typeof(Action) }); + if (method == null) + { + Assert.Fail( + $"{nameof(BrowserDrivers)} has no {StartOnceName}, so a " + + "browser's first start is spent inside a test."); + } + method.Invoke(null, new object[] { browser, start }); + } + + private static IReadOnlyDictionary FirstStarts() + { + var property = typeof(BrowserDrivers).GetProperty( + FirstStartsName, BindingFlags.Public | BindingFlags.Static); + if (property == null) + { + Assert.Fail( + $"{nameof(BrowserDrivers)} has no {FirstStartsName}, so " + + "nothing records a browser's first start."); + } + return (IReadOnlyDictionary)property.GetValue(null); + } + + private static int TimeoutSeconds() + { + var field = typeof(BrowserDrivers).GetField( + TimeoutName, BindingFlags.Public | BindingFlags.Static); + if (field == null) + { + Assert.Fail( + $"{nameof(BrowserDrivers)} has no {TimeoutName}."); + } + return (int)field.GetValue(null); + } + } +}