Demote flaky flicker tests from presubmit

The test flakiness is significantly slowing down presubmit Bug: 209843622 Test: atest FlickerTests WMShellFlickerTests Change-Id: I096e9f69e6de8ff73d527db533e76e8da44fdcb0
2022-01-22 18:23:25 +01:00
parent 2c54922c72
commit ba142e0089
9 changed files with 51 additions and 3 deletions
--- a/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/bubble/ExpandBubbleScreen.kt
+++ b/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/bubble/ExpandBubbleScreen.kt
@@ -24,6 +24,9 @@ import com.android.server.wm.flicker.FlickerParametersRunnerFactory
 import com.android.server.wm.flicker.FlickerTestParameter
 import com.android.server.wm.flicker.annotation.Group4
 import com.android.server.wm.flicker.dsl.FlickerBuilder
+import com.android.wm.shell.flicker.helpers.BaseAppHelper
+import org.junit.Assume
+import org.junit.Before
 import org.junit.runner.RunWith
 import org.junit.Test
 import org.junit.runners.Parameterized
@@ -59,6 +62,12 @@ class ExpandBubbleScreen(testSpec: FlickerTestParameter) : BaseBubbleScreen(test
            }
        }

+    @Before
+    fun setup() {
+        // This test doesn't work in shell transitions because of b/205288792
+        Assume.assumeFalse(BaseAppHelper.isShellTransitionsEnabled())
+    }
+
    @Presubmit
    @Test
    fun testAppIsAlwaysVisible() {
--- a/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/bubble/MultiBubblesScreen.kt
+++ b/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/bubble/MultiBubblesScreen.kt
@@ -25,6 +25,9 @@ import com.android.server.wm.flicker.FlickerParametersRunnerFactory
 import com.android.server.wm.flicker.FlickerTestParameter
 import com.android.server.wm.flicker.annotation.Group4
 import com.android.server.wm.flicker.dsl.FlickerBuilder
+import com.android.wm.shell.flicker.helpers.BaseAppHelper
+import org.junit.Assume
+import org.junit.Before
 import org.junit.runner.RunWith
 import org.junit.Test
 import org.junit.runners.Parameterized
@@ -67,6 +70,12 @@ class MultiBubblesScreen(testSpec: FlickerTestParameter) : BaseBubbleScreen(test
            }
        }

+    @Before
+    fun setup() {
+        // This test doesn't work in shell transitions because of b/205288792
+        Assume.assumeFalse(BaseAppHelper.isShellTransitionsEnabled())
+    }
+
    @Presubmit
    @Test
    fun testAppIsAlwaysVisible() {
--- a/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipViaExpandButtonClickTest.kt
+++ b/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipViaExpandButtonClickTest.kt
@@ -82,6 +82,11 @@ class ExitPipViaExpandButtonClickTest(
    @Test
    override fun statusBarLayerRotatesScales() = super.statusBarLayerRotatesScales()

+    /** {@inheritDoc}  */
+    @FlakyTest(bugId = 197726610)
+    @Test
+    override fun pipLayerExpands() = super.pipLayerExpands()
+
    companion object {
        /**
         * Creates the test configurations.
--- a/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipViaIntentTest.kt
+++ b/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipViaIntentTest.kt
@@ -101,6 +101,11 @@ class ExitPipViaIntentTest(testSpec: FlickerTestParameter) : ExitPipToAppTransit
    @Test
    override fun statusBarLayerRotatesScales() = super.statusBarLayerRotatesScales()

+    /** {@inheritDoc}  */
+    @FlakyTest(bugId = 197726610)
+    @Test
+    override fun pipLayerExpands() = super.pipLayerExpands()
+
    companion object {
        /**
         * Creates the test configurations.
--- a/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipWithDismissButtonTest.kt
+++ b/libs/WindowManager/Shell/tests/flicker/src/com/android/wm/shell/flicker/pip/ExitPipWithDismissButtonTest.kt
@@ -90,6 +90,11 @@ class ExitPipWithDismissButtonTest(testSpec: FlickerTestParameter) : ExitPipTran
    @Test
    override fun statusBarLayerRotatesScales() = super.statusBarLayerRotatesScales()

+    /** {@inheritDoc}  */
+    @FlakyTest(bugId = 215869110)
+    @Test
+    override fun focusDoesNotChange() = super.focusDoesNotChange()
+
    companion object {
        /**
         * Creates the test configurations.
--- a/tests/FlickerTests/src/com/android/server/wm/flicker/close/CloseAppHomeButtonTest.kt
+++ b/tests/FlickerTests/src/com/android/server/wm/flicker/close/CloseAppHomeButtonTest.kt
@@ -108,6 +108,15 @@ class CloseAppHomeButtonTest(testSpec: FlickerTestParameter) : CloseAppTransitio
        super.entireScreenCovered()
    }

+    /** {@inheritDoc} */
+    @Presubmit
+    @Test
+    override fun visibleLayersShownMoreThanOneConsecutiveEntry() {
+        // This test doesn't work in shell transitions because of b/215885246
+        assumeFalse(isShellTransitionsEnabled)
+        super.visibleLayersShownMoreThanOneConsecutiveEntry()
+    }
+
    companion object {
        /**
         * Creates the test configurations.
--- a/tests/FlickerTests/src/com/android/server/wm/flicker/ime/LaunchAppShowImeAndDialogThemeAppTest.kt
+++ b/tests/FlickerTests/src/com/android/server/wm/flicker/ime/LaunchAppShowImeAndDialogThemeAppTest.kt
@@ -20,6 +20,7 @@ import android.app.Instrumentation
 import android.platform.test.annotations.Presubmit
 import android.view.Surface
 import android.view.WindowManagerPolicyConstants
+import androidx.test.filters.FlakyTest
 import androidx.test.filters.RequiresDevice
 import androidx.test.platform.app.InstrumentationRegistry
 import com.android.server.wm.flicker.FlickerBuilderProvider
@@ -71,7 +72,7 @@ class LaunchAppShowImeAndDialogThemeAppTest(private val testSpec: FlickerTestPar
    /**
     * Checks that [FlickerComponentName.IME] layer becomes visible during the transition
     */
-    @Presubmit
+    @FlakyTest(bugId = 215884488)
    @Test
    fun imeWindowIsAlwaysVisible() = testSpec.imeWindowIsAlwaysVisible()

--- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt
+++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt
@@ -122,6 +122,11 @@ class OpenAppColdTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSp
    @Test
    override fun navBarWindowIsVisible() = super.navBarWindowIsVisible()

+    /** {@inheritDoc} */
+    @FlakyTest(bugId = 213852103)
+    @Test
+    override fun entireScreenCovered() = super.entireScreenCovered()
+
    companion object {
        /**
         * Creates the test configurations.
--- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt
+++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt
@@ -127,7 +127,7 @@ class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransiti
     * The `isAppWindowInvisible` step is optional because we log once per frame, upon logging,
     * the window may be visible or not depending on what was processed until that moment.
     */
-    @Presubmit
+    @FlakyTest(bugId = 203538234)
    @Test
    fun appWindowBecomesVisible() {
        testSpec.assertWm {
@@ -240,7 +240,7 @@ class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransiti
     * it cannot use the regular assertion (check over time), because on lock screen neither
     * the app not the launcher are visible, and there is no top visible window.
     */
-    @Presubmit
+    @FlakyTest(bugId = 203538234)
    @Test
    override fun appWindowReplacesLauncherAsTopWindow() {
        testSpec.assertWm {