Retry failed tests

.woodpecker/integration-test.yml

@@ -19,7 +19,8 @@ steps:
       - curl -fL -o adbserver-desktop.jar https://raw.githubusercontent.com/KasperskyLab/Kaspresso/v.1.6.0/artifacts/adbserver-desktop.jar
       - java -jar adbserver-desktop.jar &
       - sleep 3
-      - ./gradlew :opencloudApp:connectedOriginalDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.annotation=eu.opencloud.android.IntegrationTest --console=plain
+      - adb logcat -c
+      - trap 'adb logcat -d -s RetryRule:W | tee retries.log' EXIT; ./gradlew :opencloudApp:connectedOriginalDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.annotation=eu.opencloud.android.IntegrationTest -Pandroid.testInstrumentationRunnerArguments.retries=1 --console=plain
 
   - name: instrumented data tests
     image: *android_image
@@ -36,8 +37,9 @@ steps:
     commands:
       - |
         python3 -c "
-        import xml.etree.ElementTree as ET, glob
-        passed = skipped = failed = 0
+        import xml.etree.ElementTree as ET, glob, os
+        retries = open('retries.log').read() if os.path.exists('retries.log') else ''
+        passed = skipped = failed = flaky = 0
         for f in glob.glob('**/TEST-*.xml', recursive=True):
             root = ET.parse(f).getroot()
             suites = [root] if root.tag == 'testsuite' else root.findall('testsuite')
@@ -48,11 +50,14 @@ steps:
                         print(f'⏭  {name}'); skipped += 1
                     elif tc.find('failure') is not None:
                         print(f'❌ {name}'); failed += 1
+                    elif f\"{tc.get('name')}({suite.get('name')})\" in retries:
+                        print(f'🔁 {name} (passed on retry)'); flaky += 1
                     else:
                         print(f'✅ {name}'); passed += 1
         print()
         print(f'✅ passed:  {passed}')
         print(f'❌ failed:  {failed}')
+        print(f'🔁 flaky:   {flaky}')
         print(f'⏭  skipped: {skipped}')
-        print(f'   total:   {passed+failed+skipped}')
+        print(f'   total:   {passed+failed+flaky+skipped}')
         "

build.gradle

@@ -53,6 +53,11 @@ subprojects {
         }
     }
 
+    // Hide Kotlin compiler warnings in CI logs
+    tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
+        if (System.getenv("CI")) kotlinOptions.suppressWarnings = true
+    }
+
     plugins.withId("com.android.application") {
         android {
             buildTypes {

opencloudApp/src/integrationTest/java/eu/opencloud/android/LoginTest.kt

@@ -22,6 +22,12 @@ class LoginTest : TestCase(
         )
     }
 ) {
+    @get:Rule(order = Int.MIN_VALUE)
+    val retryRule = RetryRule()
+
+    @get:Rule(order = Int.MIN_VALUE + 1)
+    val removeAccountsRule = RemoveAccountsRule()
+
     @get:Rule
     val grantPermissionRule: GrantPermissionRule = GrantPermissionRule.grant(
         android.Manifest.permission.POST_NOTIFICATIONS

opencloudApp/src/integrationTest/java/eu/opencloud/android/RemoveAccountsRule.kt

@@ -0,0 +1,16 @@
+package eu.opencloud.android
+
+import android.accounts.AccountManager
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.rules.ExternalResource
+
+/** Removes all app accounts so every attempt starts from the start screen. */
+class RemoveAccountsRule : ExternalResource() {
+    override fun before() {
+        val context = InstrumentationRegistry.getInstrumentation().targetContext
+        val accountManager = AccountManager.get(context)
+        accountManager.getAccountsByType(context.getString(R.string.account_type)).forEach {
+            accountManager.removeAccountExplicitly(it)
+        }
+    }
+}

opencloudApp/src/integrationTest/java/eu/opencloud/android/RetryRule.kt

@@ -0,0 +1,28 @@
+package eu.opencloud.android
+
+import android.util.Log
+import androidx.test.platform.app.InstrumentationRegistry
+import org.junit.rules.TestRule
+import org.junit.runner.Description
+import org.junit.runners.model.Statement
+
+/**
+ * Re-runs a failed test. Retries come from the "retries" instrumentation argument (default 0).
+ * Must be the outermost rule.
+ */
+class RetryRule(
+    private val retries: Int = InstrumentationRegistry.getArguments().getString("retries")?.toIntOrNull() ?: 0
+) : TestRule {
+    override fun apply(base: Statement, description: Description) = object : Statement() {
+        override fun evaluate() {
+            repeat(retries) {
+                try {
+                    return base.evaluate()
+                } catch (t: Throwable) {
+                    Log.w("RetryRule", "${description.displayName} failed, retrying", t)
+                }
+            }
+            base.evaluate()
+        }
+    }
+}