From 44fc92a7be6f21bdc1e925f1d91597beeaa8a89b Mon Sep 17 00:00:00 2001 From: Nataniel Borges Date: Mon, 30 Aug 2021 15:24:01 +0200 Subject: [PATCH] Update app launch tests This CL: Promotes stable assertions from flaky to postsubmit Updates test documentation Fixes wait condition on app launch from locked screen Bug: 198123259 Bug: 198123260 Test: atest FlickerTests Change-Id: I0ec127c043d67f2fddada53ea1555008f997b747 --- .../wm/flicker/launch/OpenAppColdTest.kt | 55 ++++++-- .../flicker/launch/OpenAppFromOverviewTest.kt | 52 +++++++- .../launch/OpenAppNonResizeableTest.kt | 123 +++++++++++++++--- .../wm/flicker/launch/OpenAppTransition.kt | 64 +++++++-- .../wm/flicker/launch/OpenAppWarmTest.kt | 47 ++++++- 5 files changed, 302 insertions(+), 39 deletions(-) diff --git a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt index e6dc8523acbf1..be919cd67c1e5 100644 --- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt +++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppColdTest.kt @@ -16,6 +16,8 @@ package com.android.server.wm.flicker.launch +import android.platform.test.annotations.Postsubmit +import android.platform.test.annotations.Presubmit import androidx.test.filters.FlakyTest import androidx.test.filters.RequiresDevice import com.android.server.wm.flicker.FlickerParametersRunnerFactory @@ -33,8 +35,21 @@ import org.junit.runners.MethodSorters import org.junit.runners.Parameterized /** - * Test cold launch app from launcher. + * Test cold launching an app from launcher + * * To run this test: `atest FlickerTests:OpenAppColdTest` + * + * Actions: + * Make sure no apps are running on the device + * Launch an app [testApp] and wait animation to complete + * + * Notes: + * 1. Some default assertions (e.g., nav bar, status bar and screen covered) + * are inherited [OpenAppTransition] + * 2. Part of the test setup occurs automatically via + * [com.android.server.wm.flicker.TransitionRunnerWithRules], + * including configuring navigation mode, initial orientation and ensuring no + * apps are running before setup */ @RequiresDevice @RunWith(Parameterized::class) @@ -42,6 +57,9 @@ import org.junit.runners.Parameterized @FixMethodOrder(MethodSorters.NAME_ASCENDING) @Group1 class OpenAppColdTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSpec) { + /** + * Defines the transition used to run the test + */ override val transition: FlickerBuilder.(Map) -> Unit get() = { super.transition(this, it) @@ -62,25 +80,46 @@ class OpenAppColdTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSp } } + /** {@inheritDoc} */ @FlakyTest @Test override fun navBarLayerRotatesAndScales() { super.navBarLayerRotatesAndScales() } - @FlakyTest(bugId = 192721431) + /** {@inheritDoc} */ + @Postsubmit @Test - override fun appLayerReplacesLauncher() { - super.appLayerReplacesLauncher() - } + override fun appLayerReplacesLauncher() = super.appLayerReplacesLauncher() - @FlakyTest(bugId = 192721431) + /** {@inheritDoc} */ + @Postsubmit @Test - override fun appWindowReplacesLauncherAsTopWindow() { + override fun appWindowReplacesLauncherAsTopWindow() = super.appWindowReplacesLauncherAsTopWindow() - } + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun launcherWindowBecomesInvisible() = super.launcherWindowBecomesInvisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarLayerIsVisible() = super.navBarLayerIsVisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarWindowIsVisible() = super.navBarWindowIsVisible() companion object { + /** + * Creates the test configurations. + * + * See [FlickerTestParameterFactory.getConfigNonRotationTests] for configuring + * repetitions, screen orientation and navigation modes. + */ @Parameterized.Parameters(name = "{0}") @JvmStatic fun getParams(): Collection { diff --git a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppFromOverviewTest.kt b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppFromOverviewTest.kt index 7833e2f25eec6..3678f33aa46ec 100644 --- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppFromOverviewTest.kt +++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppFromOverviewTest.kt @@ -16,6 +16,7 @@ package com.android.server.wm.flicker.launch +import android.platform.test.annotations.Presubmit import androidx.test.filters.FlakyTest import androidx.test.filters.RequiresDevice import com.android.server.wm.flicker.FlickerParametersRunnerFactory @@ -33,8 +34,23 @@ import org.junit.runners.MethodSorters import org.junit.runners.Parameterized /** - * Launch an app from the recents app view (the overview) + * Test launching an app from the recents app view (the overview) + * * To run this test: `atest FlickerTests:OpenAppFromOverviewTest` + * + * Actions: + * Launch [testApp] + * Press recents + * Relaunch an app [testApp] by selecting it in the overview screen, and wait animation to + * complete (only this action is traced) + * + * Notes: + * 1. Some default assertions (e.g., nav bar, status bar and screen covered) + * are inherited [OpenAppTransition] + * 2. Part of the test setup occurs automatically via + * [com.android.server.wm.flicker.TransitionRunnerWithRules], + * including configuring navigation mode, initial orientation and ensuring no + * apps are running before setup */ @RequiresDevice @RunWith(Parameterized::class) @@ -42,6 +58,9 @@ import org.junit.runners.Parameterized @FixMethodOrder(MethodSorters.NAME_ASCENDING) @Group1 class OpenAppFromOverviewTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSpec) { + /** + * Defines the transition used to run the test + */ override val transition: FlickerBuilder.(Map) -> Unit get() = { super.transition(this, it) @@ -63,13 +82,38 @@ class OpenAppFromOverviewTest(testSpec: FlickerTestParameter) : OpenAppTransitio } } + /** {@inheritDoc} */ @FlakyTest @Test - override fun navBarLayerRotatesAndScales() { - super.navBarLayerRotatesAndScales() - } + override fun navBarLayerRotatesAndScales() = super.navBarLayerRotatesAndScales() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun appLayerReplacesLauncher() = super.appLayerReplacesLauncher() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun launcherWindowBecomesInvisible() = super.launcherWindowBecomesInvisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarLayerIsVisible() = super.navBarLayerIsVisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarWindowIsVisible() = super.navBarWindowIsVisible() companion object { + /** + * Creates the test configurations. + * + * See [FlickerTestParameterFactory.getConfigNonRotationTests] for configuring + * repetitions, screen orientation and navigation modes. + */ @Parameterized.Parameters(name = "{0}") @JvmStatic fun getParams(): Collection { diff --git a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt index 93a58be96091b..b7176122095be 100644 --- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt +++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppNonResizeableTest.kt @@ -16,6 +16,7 @@ package com.android.server.wm.flicker.launch +import android.content.ComponentName import android.platform.test.annotations.Postsubmit import android.platform.test.annotations.Presubmit import android.view.Surface @@ -29,6 +30,7 @@ import com.android.server.wm.flicker.annotation.Group1 import com.android.server.wm.flicker.helpers.NonResizeableAppHelper import com.android.server.wm.flicker.dsl.FlickerBuilder import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper +import com.google.common.truth.Truth import org.junit.FixMethodOrder import org.junit.Test import org.junit.runner.RunWith @@ -36,8 +38,21 @@ import org.junit.runners.MethodSorters import org.junit.runners.Parameterized /** - * Launch an app while the phone is locked + * Test launching an app while the device is locked + * * To run this test: `atest FlickerTests:OpenAppNonResizeableTest` + * + * Actions: + * Lock the device. + * Launch an app on top of the lock screen [testApp] and wait animation to complete + * + * Notes: + * 1. Some default assertions (e.g., nav bar, status bar and screen covered) + * are inherited [OpenAppTransition] + * 2. Part of the test setup occurs automatically via + * [com.android.server.wm.flicker.TransitionRunnerWithRules], + * including configuring navigation mode, initial orientation and ensuring no + * apps are running before setup */ @RequiresDevice @RunWith(Parameterized::class) @@ -46,14 +61,20 @@ import org.junit.runners.Parameterized @Group1 class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSpec) { override val testApp = NonResizeableAppHelper(instrumentation) + private val colorFadComponent = ComponentName("", "ColorFade BLAST#") + /** + * Defines the transition used to run the test + */ override val transition: FlickerBuilder.(Map) -> Unit - get() = { - super.transition(this, it) + get() = { args -> + super.transition(this, args) setup { eachRun { device.sleep() - wmHelper.waitForAppTransitionIdle() + wmHelper.waitFor("noAppWindowsOnTop") { + it.wmState.topVisibleAppWindow.isEmpty() + } } } teardown { @@ -67,17 +88,29 @@ class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransiti } } + /** + * Checks that the nav bar layer starts visible, becomes invisible during unlocking animation + * and becomes visible at the end + */ @Presubmit @Test - override fun navBarLayerIsVisible() { - testSpec.assertLayersEnd { - isVisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + fun navBarLayerVisibilityChanges() { + testSpec.assertLayers { + this.isVisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + .then() + .isInvisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + .then() + .isVisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) } } + /** + * Checks that the app layer doesn't exist at the start of the transition, that it is + * created (invisible) and becomes visible during the transition + */ @Presubmit @Test - fun nonResizableAppLayerBecomesVisible() { + fun appLayerBecomesVisible() { testSpec.assertLayers { this.notContains(testApp.component) .then() @@ -87,9 +120,16 @@ class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransiti } } + /** + * Checks that the app window doesn't exist at the start of the transition, that it is + * created (invisible - optional) and becomes visible during the transition + * + * The `isAppWindowInvisible` step is optional because we log once per frame, upon logging, + * the window may be visible or not depending on what was processed until that moment. + */ @Presubmit @Test - fun nonResizableAppWindowBecomesVisible() { + fun appWindowBecomesVisible() { testSpec.assertWm { this.notContains(testApp.component) .then() @@ -100,46 +140,95 @@ class OpenAppNonResizeableTest(testSpec: FlickerTestParameter) : OpenAppTransiti } } + /** + * Checks if [testApp] is visible at the end of the transition + */ @Presubmit @Test - fun nonResizableAppWindowBecomesVisibleAtEnd() { + fun appWindowBecomesVisibleAtEnd() { testSpec.assertWmEnd { this.isVisible(testApp.component) } } - @FlakyTest + /** + * Checks that the nav bar starts the transition visible, then becomes invisible during + * then unlocking animation and becomes visible at the end of the transition + */ + @Postsubmit @Test - override fun navBarWindowIsVisible() = super.navBarWindowIsVisible() + fun navBarWindowsVisibilityChanges() { + testSpec.assertWm { + this.isAboveAppWindowVisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + .then() + .isNonAppWindowInvisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + .then() + .isAboveAppWindowVisible(WindowManagerStateHelper.NAV_BAR_COMPONENT) + } + } + /** {@inheritDoc} */ @FlakyTest @Test override fun visibleWindowsShownMoreThanOneConsecutiveEntry() = super.visibleWindowsShownMoreThanOneConsecutiveEntry() + /** {@inheritDoc} */ @FlakyTest @Test override fun visibleLayersShownMoreThanOneConsecutiveEntry() = super.visibleLayersShownMoreThanOneConsecutiveEntry() + /** {@inheritDoc} */ @Postsubmit @Test override fun entireScreenCovered() = super.entireScreenCovered() + /** + * Checks that the focus changes from the launcher to [testApp] + */ @FlakyTest @Test override fun focusChanges() = super.focusChanges() - @FlakyTest + /** + * Checks that the screen is locked at the start of the transition ([colorFadComponent]) + * layer is visible + */ + @Postsubmit @Test - override fun appLayerReplacesLauncher() = super.appLayerReplacesLauncher() + fun screenLockedStart() { + testSpec.assertLayersStart { + isVisible(colorFadComponent) + } + } - @FlakyTest + /** + * This test checks if the launcher is visible at the start and the app at the end, + * it cannot use the regular assertion (check over time), because on lock screen neither + * the app not the launcher are visible, and there is no top visible window. + */ + @Postsubmit @Test - override fun appWindowReplacesLauncherAsTopWindow() = - super.appWindowReplacesLauncherAsTopWindow() + override fun appWindowReplacesLauncherAsTopWindow() { + testSpec.assertWm { + this.invoke("noAppWindowsOnTop") { + Truth.assertWithMessage("Should not have any app window on top " + + "when the screen is locked") + .that(it.wmState.topVisibleAppWindow) + .isEmpty() + }.then() + .isAppWindowOnTop(testApp.component) + } + } companion object { + /** + * Creates the test configurations. + * + * See [FlickerTestParameterFactory.getConfigNonRotationTests] for configuring + * repetitions, screen orientation and navigation modes. + */ @Parameterized.Parameters(name = "{0}") @JvmStatic fun getParams(): Collection { diff --git a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppTransition.kt b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppTransition.kt index 434b244816581..14d17f82b8052 100644 --- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppTransition.kt +++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppTransition.kt @@ -40,12 +40,16 @@ import com.android.server.wm.flicker.statusBarLayerIsVisible import com.android.server.wm.flicker.statusBarLayerRotatesScales import com.android.server.wm.flicker.statusBarWindowIsVisible import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper.Companion.SNAPSHOT_COMPONENT +import com.android.server.wm.traces.parser.windowmanager.WindowManagerStateHelper.Companion.SPLASH_SCREEN_COMPONENT import org.junit.Test abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { protected val instrumentation: Instrumentation = InstrumentationRegistry.getInstrumentation() protected open val testApp: StandardAppHelper = SimpleAppHelper(instrumentation) + /** + * Defines the transition used to run the test + */ protected open val transition: FlickerBuilder.(Map) -> Unit = { withTestName { testSpec.name } repeat { testSpec.config.repetitions } @@ -62,6 +66,10 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { } } + /** + * Entry point for the test runner. It will use this method to initialize and cache + * flicker executions + */ @FlickerBuilderProvider fun buildFlicker(): FlickerBuilder { return FlickerBuilder(instrumentation).apply { @@ -69,42 +77,60 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { } } - @Presubmit - @Test + /** + * Checks that the navigation bar window is visible during the whole transition + */ open fun navBarWindowIsVisible() { testSpec.navBarWindowIsVisible() } - @Presubmit - @Test + /** + * Checks that the navigation bar layer is visible during the whole transition + */ open fun navBarLayerIsVisible() { testSpec.navBarLayerIsVisible() } + /** + * Checks the position of the navigation bar at the start and end of the transition + */ @Presubmit @Test open fun navBarLayerRotatesAndScales() { testSpec.navBarLayerRotatesAndScales(Surface.ROTATION_0, testSpec.config.endRotation) } + /** + * Checks that the status bar window is visible during the whole transition + */ @Presubmit @Test open fun statusBarWindowIsVisible() { testSpec.statusBarWindowIsVisible() } + /** + * Checks that the status bar layer is visible during the whole transition + */ @Presubmit @Test open fun statusBarLayerIsVisible() { testSpec.statusBarLayerIsVisible() } + /** + * Checks the position of the status bar at the start and end of the transition + */ @Presubmit @Test open fun statusBarLayerRotatesScales() { testSpec.statusBarLayerRotatesScales(Surface.ROTATION_0, testSpec.config.endRotation) } + /** + * Checks that all windows that are visible on the trace, are visible for at least 2 + * consecutive entries. + */ @Presubmit @Test open fun visibleWindowsShownMoreThanOneConsecutiveEntry() { @@ -113,6 +139,10 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { } } + /** + * Checks that all layers that are visible on the trace, are visible for at least 2 + * consecutive entries. + */ @Presubmit @Test open fun visibleLayersShownMoreThanOneConsecutiveEntry() { @@ -121,11 +151,16 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { } } + /** + * Checks that all parts of the screen are covered during the transition + */ @Presubmit @Test - // During testing the launcher is always in portrait mode open fun entireScreenCovered() = testSpec.entireScreenCovered() + /** + * Checks that the focus changes from the launcher to [testApp] + */ @Presubmit @Test open fun focusChanges() { @@ -134,12 +169,19 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { } } - @Presubmit - @Test + /** + * Checks that [LAUNCHER_COMPONENT] layer is visible at the start of the transition, and + * is replaced by [testApp], which remains visible until the end + */ open fun appLayerReplacesLauncher() { testSpec.replacesLayer(LAUNCHER_COMPONENT, testApp.component) } + /** + * Checks that [LAUNCHER_COMPONENT] window is visible at the start of the transition, and + * is replaced by a snapshot or splash screen (optional), and finally, is replaced by + * [testApp], which remains visible until the end + */ @Presubmit @Test open fun appWindowReplacesLauncherAsTopWindow() { @@ -148,12 +190,16 @@ abstract class OpenAppTransition(protected val testSpec: FlickerTestParameter) { .then() .isAppWindowOnTop(SNAPSHOT_COMPONENT, isOptional = true) .then() + .isAppWindowOnTop(SPLASH_SCREEN_COMPONENT, isOptional = true) + .then() .isAppWindowOnTop(testApp.component) } } - @Presubmit - @Test + /** + * Checks that [LAUNCHER_COMPONENT] window is visible at the start, and + * becomes invisible during the transition + */ open fun launcherWindowBecomesInvisible() { testSpec.assertWm { this.isAppWindowVisible(LAUNCHER_COMPONENT) diff --git a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppWarmTest.kt b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppWarmTest.kt index b509c61d2aba0..5edee0cf0ca0c 100644 --- a/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppWarmTest.kt +++ b/tests/FlickerTests/src/com/android/server/wm/flicker/launch/OpenAppWarmTest.kt @@ -16,6 +16,7 @@ package com.android.server.wm.flicker.launch +import android.platform.test.annotations.Presubmit import androidx.test.filters.FlakyTest import androidx.test.filters.RequiresDevice import com.android.server.wm.flicker.FlickerParametersRunnerFactory @@ -32,8 +33,22 @@ import org.junit.runners.MethodSorters import org.junit.runners.Parameterized /** - * Test warm launch app. + * Test warm launching an app from launcher + * * To run this test: `atest FlickerTests:OpenAppWarmTest` + * + * Actions: + * Launch [testApp] + * Press home + * Relaunch an app [testApp] and wait animation to complete (only this action is traced) + * + * Notes: + * 1. Some default assertions (e.g., nav bar, status bar and screen covered) + * are inherited [OpenAppTransition] + * 2. Part of the test setup occurs automatically via + * [com.android.server.wm.flicker.TransitionRunnerWithRules], + * including configuring navigation mode, initial orientation and ensuring no + * apps are running before setup */ @RequiresDevice @RunWith(Parameterized::class) @@ -41,6 +56,9 @@ import org.junit.runners.Parameterized @FixMethodOrder(MethodSorters.NAME_ASCENDING) @Group1 class OpenAppWarmTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSpec) { + /** + * Defines the transition used to run the test + */ override val transition: FlickerBuilder.(Map) -> Unit get() = { super.transition(this, it) @@ -65,11 +83,38 @@ class OpenAppWarmTest(testSpec: FlickerTestParameter) : OpenAppTransition(testSp } } + /** {@inheritDoc} */ @FlakyTest @Test override fun navBarLayerRotatesAndScales() = super.navBarLayerRotatesAndScales() + /** {@inheritDoc} */ + @Presubmit + @Test + override fun appLayerReplacesLauncher() = super.appLayerReplacesLauncher() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun launcherWindowBecomesInvisible() = super.launcherWindowBecomesInvisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarLayerIsVisible() = super.navBarLayerIsVisible() + + /** {@inheritDoc} */ + @Presubmit + @Test + override fun navBarWindowIsVisible() = super.navBarWindowIsVisible() + companion object { + /** + * Creates the test configurations. + * + * See [FlickerTestParameterFactory.getConfigNonRotationTests] for configuring + * repetitions, screen orientation and navigation modes. + */ @Parameterized.Parameters(name = "{0}") @JvmStatic fun getParams(): Collection {