Start Chrome and Firefox once before the first test that uses them

Helpers/BrowserDrivers.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
 using OpenQA.Selenium;
@@ -65,6 +66,22 @@ public static class BrowserDrivers
         /// </summary>
         public const string EdgeDriverVariable = "EDGEWEBDRIVER";
 
+        /// <summary>
+        /// Seconds the first start of a browser in a run may take. A
+        /// browser's first start on a fresh runner can take minutes, where
+        /// the driver allows each command 60 seconds, so the first start is
+        /// made by <see cref="StartOnce"/> with this limit instead.
+        /// </summary>
+        public const int FirstStartTimeoutSeconds = 300;
+
+        /// <summary>Page the first start of a browser loads.</summary>
+        private const string FirstStartPage = "about:blank";
+
+        private static readonly object _firstStartLock = new object();
+
+        private static readonly Dictionary<string, string> _firstStarts =
+            new Dictionary<string, string>();
+
         /// <summary>Path to the Chrome or Chromium binary to drive.</summary>
         public const string ChromeBinaryVariable = "CHROME_BIN";
 
@@ -125,6 +142,12 @@ public static WebDriver CreateChrome(ChromeOptions options)
             }
             SetBinaryLocation(options, ChromeBinaryVariable, _chromeBinaryNames);
             var driverPath = FindDriver(ChromeDriverVariable, ChromeDriverNames);
+            StartOnce("Chrome", timeout => Visit(new ChromeDriver(
+                driverPath != null
+                    ? ChromeDriverService.CreateDefaultService(driverPath)
+                    : ChromeDriverService.CreateDefaultService(),
+                options,
+                timeout)));
             if (driverPath != null)
             {
                 return new ChromeDriver(
@@ -154,6 +177,12 @@ public static WebDriver CreateFirefox(FirefoxOptions options)
             SetBinaryLocation(
                 options, FirefoxBinaryVariable, _firefoxBinaryNames);
             var driverPath = FindDriver(GeckoDriverVariable, GeckoDriverNames);
+            StartOnce("Firefox", timeout => Visit(new FirefoxDriver(
+                driverPath != null
+                    ? FirefoxDriverService.CreateDefaultService(driverPath)
+                    : FirefoxDriverService.CreateDefaultService(),
+                options,
+                timeout)));
             if (driverPath != null)
             {
                 return new FirefoxDriver(
@@ -166,7 +195,10 @@ public static WebDriver CreateFirefox(FirefoxOptions options)
         }
 
         /// <summary>
-        /// Creates an Edge driver.
+        /// Creates an Edge driver. Edge is not started once first, as Chrome
+        /// and Firefox are, because headless Edge on the Ubuntu runners hangs
+        /// on its first page, which would hold the run for the whole of
+        /// <see cref="FirstStartTimeoutSeconds"/>.
         /// </summary>
         /// <param name="options">Options for the browser.</param>
         /// <returns>A driver, which the caller quits.</returns>
@@ -201,6 +233,102 @@ public static WebDriver CreateEdge(EdgeOptions options)
                 "Edge", EdgeDriverVariable, EdgeDriverNames);
         }
 
+        /// <summary>
+        /// The line written at the first start of each browser in this run,
+        /// by the browser's name.
+        /// </summary>
+        public static IReadOnlyDictionary<string, string> FirstStarts
+        {
+            get
+            {
+                lock (_firstStartLock)
+                {
+                    return new Dictionary<string, string>(_firstStarts);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Runs <paramref name="start"/> the first time a browser is asked
+        /// for in this run, and never again, so that the browser's slow first
+        /// start is made before the first test that uses it and not inside
+        /// one of that test's commands. Writes how long it took to the output
+        /// of the test that asked. Whatever stops the start is written and
+        /// not thrown, and the test then meets it when it starts the browser
+        /// itself.
+        /// </summary>
+        /// <param name="browser">Name of the browser.</param>
+        /// <param name="start">
+        /// Starts the browser, loads a page and quits, with the time limit
+        /// for each command it is given.
+        /// </param>
+        public static void StartOnce(string browser, Action<TimeSpan> start)
+        {
+            if (browser == null)
+            {
+                throw new ArgumentNullException(nameof(browser));
+            }
+            if (start == null)
+            {
+                throw new ArgumentNullException(nameof(start));
+            }
+            lock (_firstStartLock)
+            {
+                if (_firstStarts.ContainsKey(browser))
+                {
+                    return;
+                }
+                var watch = Stopwatch.StartNew();
+                string line;
+                try
+                {
+                    start(TimeSpan.FromSeconds(FirstStartTimeoutSeconds));
+                    line = $"Started {browser} once before its first test "
+                        + "in " + Seconds(watch) + " s.";
+                }
+                catch (Exception e)
+                {
+                    line = $"{browser} did not finish its first start in "
+                        + Seconds(watch) + " s, so the test starts it as it "
+                        + "would have. " + FirstLine(e.Message);
+                }
+                _firstStarts[browser] = line;
+                Console.WriteLine(line);
+            }
+        }
+
+        /// <summary>
+        /// Loads a page in <paramref name="driver"/> and quits it.
+        /// </summary>
+        /// <param name="driver">A driver just created.</param>
+        private static void Visit(WebDriver driver)
+        {
+            try
+            {
+                driver.Navigate().GoToUrl(FirstStartPage);
+            }
+            finally
+            {
+                driver.Quit();
+            }
+        }
+
+        /// <summary>Seconds on the watch, to one decimal place.</summary>
+        /// <param name="watch">A running watch.</param>
+        /// <returns>The seconds, written the same in every culture.</returns>
+        private static string Seconds(Stopwatch watch) =>
+            watch.Elapsed.TotalSeconds.ToString(
+                "0.0", System.Globalization.CultureInfo.InvariantCulture);
+
+        /// <summary>
+        /// The first line of <paramref name="text"/>, which for a driver's
+        /// message is the reason without the details that follow it.
+        /// </summary>
+        /// <param name="text">Text of one or more lines.</param>
+        /// <returns>The first line, or an empty string.</returns>
+        private static string FirstLine(string text) =>
+            (text ?? string.Empty).Split('\n')[0].TrimEnd('\r');
+
         /// <summary>
         /// Runs <paramref name="create"/>, which leaves Selenium Manager to
         /// fetch a driver, and on failure says what was looked for here first,

README.md

@@ -34,7 +34,11 @@ Select a subset with `--filter TestCategory=Contract` or
 - **Browser** - a driver the machine already provides is used first, named by
   `CHROMEWEBDRIVER`, `GECKOWEBDRIVER` or `EDGEWEBDRIVER` or found on the path.
   When the machine provides none, Selenium Manager fetches one. Set
-  `SELENIUM_URL` to drive a browser on a Selenium grid instead.
+  `SELENIUM_URL` to drive a browser on a Selenium grid instead. Chrome and
+  Firefox are each started once, before the first test that uses them, with
+  up to 300 seconds for that first start, because a browser's first start on a
+  fresh runner can take longer than the 60 seconds a test's command is given.
+  The time it took is written to that test's output.
 
 ## Configuration
 

Selenium/FirstStartTests.cs

@@ -0,0 +1,199 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Reflection;
+using System.Threading;
+using FiftyOne.Pipeline.Cloud.SeleniumTests.Helpers;
+using FiftyOne.Pipeline.Cloud.Tests.Common.Helpers;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpenQA.Selenium;
+using OpenQA.Selenium.Chrome;
+using OpenQA.Selenium.Firefox;
+
+namespace FiftyOne.Pipeline.Cloud.SeleniumTests.Selenium
+{
+    /// <summary>
+    /// Checks that each browser is started once before the first test that
+    /// uses it, so that a slow first start is not spent inside a test.
+    /// </summary>
+    /// <remarks>
+    /// <see cref="BrowserDrivers"/> is reached by name, so that a build
+    /// without the start once fails these tests rather than the build.
+    /// </remarks>
+    [TestClass]
+    public class FirstStartTests
+    {
+        private const string StartOnceName = "StartOnce";
+        private const string FirstStartsName = "FirstStarts";
+        private const string TimeoutName = "FirstStartTimeoutSeconds";
+
+        /// <summary>
+        /// A browser's start is run once, however many times it is asked
+        /// for, and is given the first start's time limit.
+        /// </summary>
+        [TestMethod]
+        public void StartOnce_StartsEachBrowserOnce()
+        {
+            var browser = "Stand-in " + Guid.NewGuid();
+            var starts = 0;
+            var limit = TimeSpan.Zero;
+            Action<TimeSpan> start = timeout =>
+            {
+                starts++;
+                limit = timeout;
+            };
+
+            StartOnce(browser, start);
+            StartOnce(browser, start);
+
+            Assert.AreEqual(1, starts);
+            Assert.AreEqual(
+                TimeSpan.FromSeconds(TimeoutSeconds()),
+                limit);
+            StringAssert.StartsWith(
+                FirstStarts()[browser],
+                $"Started {browser} once before its first test in ");
+        }
+
+        /// <summary>
+        /// A first start that fails is reported and not thrown, and is not
+        /// tried again, so the test that follows meets the failure itself.
+        /// </summary>
+        [TestMethod]
+        public void StartOnce_AStartThatFails_IsReportedAndNotThrown()
+        {
+            var browser = "Stand-in " + Guid.NewGuid();
+            var starts = 0;
+            Action<TimeSpan> start = timeout =>
+            {
+                starts++;
+                throw new WebDriverException(
+                    "session not created\nsecond line");
+            };
+
+            StartOnce(browser, start);
+            StartOnce(browser, start);
+
+            Assert.AreEqual(1, starts);
+            var line = FirstStarts()[browser];
+            StringAssert.StartsWith(
+                line, $"{browser} did not finish its first start in ");
+            StringAssert.EndsWith(line, "session not created");
+        }
+
+        /// <summary>
+        /// Creating a Chrome driver starts Chrome once first.
+        /// </summary>
+        [TestMethod, TestCategory("Browser")]
+        public void Chrome_StartsOnceBeforeItsFirstTest()
+        {
+            var options = new ChromeOptions();
+            options.AddArgument("--headless");
+            options.AddArgument("--no-sandbox");
+            options.AddArgument("--disable-dev-shm-usage");
+            AssertStartedOnce(
+                "Chrome", () => BrowserDrivers.CreateChrome(options));
+        }
+
+        /// <summary>
+        /// Creating a Firefox driver starts Firefox once first.
+        /// </summary>
+        [TestMethod, TestCategory("Browser")]
+        public void Firefox_StartsOnceBeforeItsFirstTest()
+        {
+            var options = new FirefoxOptions();
+            options.AddArgument("--headless");
+            AssertStartedOnce(
+                "Firefox", () => BrowserDrivers.CreateFirefox(options));
+        }
+
+        /// <summary>
+        /// Creates the browser twice, loading a page each time, writes how
+        /// long each took, and checks the first start was made and kept.
+        /// </summary>
+        /// <param name="browser">Name the first start is kept under.</param>
+        /// <param name="create">Creates the driver.</param>
+        private static void AssertStartedOnce(
+            string browser, Func<WebDriver> create)
+        {
+            var url =
+                $"http://localhost:{TestHelpers.GetRandomUnusedPort()}/";
+            using var cts = new CancellationTokenSource();
+            var server = TestHelpers.SimpleListener(
+                url, "<!DOCTYPE html><title>First start</title>", cts.Token);
+            try
+            {
+                var times = new List<double>();
+                for (var i = 0; i < 2; i++)
+                {
+                    var watch = Stopwatch.StartNew();
+                    var driver = create();
+                    try
+                    {
+                        driver.Navigate().GoToUrl(url);
+                    }
+                    finally
+                    {
+                        driver.Quit();
+                    }
+                    times.Add(watch.Elapsed.TotalSeconds);
+                }
+                var starts = FirstStarts();
+                Assert.IsTrue(
+                    starts.ContainsKey(browser),
+                    $"{browser} was not started once before its first test.");
+                Console.WriteLine(starts[browser]);
+                Console.WriteLine(FormattableString.Invariant(
+                    $"{browser} then started and loaded a page in ")
+                    + FormattableString.Invariant(
+                        $"{times[0]:0.0} s and {times[1]:0.0} s."));
+            }
+            finally
+            {
+                cts.Cancel();
+                server?.Listener?.Stop();
+                server?.Listener?.Close();
+            }
+        }
+
+        private static void StartOnce(string browser, Action<TimeSpan> start)
+        {
+            var method = typeof(BrowserDrivers).GetMethod(
+                StartOnceName,
+                BindingFlags.Public | BindingFlags.Static,
+                new[] { typeof(string), typeof(Action<TimeSpan>) });
+            if (method == null)
+            {
+                Assert.Fail(
+                    $"{nameof(BrowserDrivers)} has no {StartOnceName}, so a "
+                    + "browser's first start is spent inside a test.");
+            }
+            method.Invoke(null, new object[] { browser, start });
+        }
+
+        private static IReadOnlyDictionary<string, string> FirstStarts()
+        {
+            var property = typeof(BrowserDrivers).GetProperty(
+                FirstStartsName, BindingFlags.Public | BindingFlags.Static);
+            if (property == null)
+            {
+                Assert.Fail(
+                    $"{nameof(BrowserDrivers)} has no {FirstStartsName}, so "
+                    + "nothing records a browser's first start.");
+            }
+            return (IReadOnlyDictionary<string, string>)property.GetValue(null);
+        }
+
+        private static int TimeoutSeconds()
+        {
+            var field = typeof(BrowserDrivers).GetField(
+                TimeoutName, BindingFlags.Public | BindingFlags.Static);
+            if (field == null)
+            {
+                Assert.Fail(
+                    $"{nameof(BrowserDrivers)} has no {TimeoutName}.");
+            }
+            return (int)field.GetValue(null);
+        }
+    }
+}