TECH: Screenshot comparison tests

allure-support/build.gradle.kts

@@ -17,6 +17,7 @@ dependencies {
     api(libs.bundles.allure)
 
     implementation(projects.kaspresso)
+    implementation(projects.adbServer.adbServerCommon)
 
     implementation(libs.kotlinStdlib)
     implementation(libs.truth)

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt

@@ -1,5 +1,6 @@
 package com.kaspersky.components.alluresupport
 
+import com.kaspersky.adbserver.common.log.logger.LogLevel
 import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
 import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider
 import com.kaspersky.components.alluresupport.files.resources.impl.DefaultAllureResourcesRootDirsProvider
@@ -10,17 +11,26 @@ import com.kaspersky.components.alluresupport.interceptors.testrun.DumpViewsTest
 import com.kaspersky.components.alluresupport.interceptors.testrun.HackyVideoRecordingTestInterceptor
 import com.kaspersky.components.alluresupport.interceptors.testrun.ScreenshotTestInterceptor
 import com.kaspersky.components.alluresupport.interceptors.testrun.VideoRecordingTestInterceptor
+import com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor
 import com.kaspersky.components.alluresupport.results.AllureResultsHack
 import com.kaspersky.components.alluresupport.runlisteners.AllureRunListener
+import com.kaspersky.components.alluresupport.visual.AllureScreenshotsComparator
+import com.kaspersky.components.alluresupport.visual.AllureVisualTestWatcher
+import com.kaspersky.kaspresso.BuildConfig
+import com.kaspersky.kaspresso.device.files.FilesImpl
+import com.kaspersky.kaspresso.device.server.AdbServerImpl
 import com.kaspersky.kaspresso.files.dirs.DefaultDirsProvider
 import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFileNamesProvider
 import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFilesProvider
 import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirNameProvider
 import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirsProvider
 import com.kaspersky.kaspresso.instrumental.InstrumentalDependencyProvider
 import com.kaspersky.kaspresso.kaspresso.Kaspresso
+import com.kaspersky.kaspresso.logger.UiTestLoggerImpl
 import com.kaspersky.kaspresso.runner.listener.addUniqueListener
 import com.kaspersky.kaspresso.runner.listener.getUniqueListener
+import com.kaspersky.kaspresso.visual.VisualTestParams
+import com.kaspersky.kaspresso.visual.VisualTestType
 
 /**
  * Kaspresso Builder that includes all appropriate interceptors to support rich Allure reports.
@@ -64,6 +74,7 @@ fun Kaspresso.Builder.addAllureSupport(): Kaspresso.Builder = apply {
  */
 fun Kaspresso.Builder.Companion.withForcedAllureSupport(
     shouldRecordVideo: Boolean = true,
+    visualTestParams: VisualTestParams = VisualTestParams(testType = VisualTestType.valueOf(BuildConfig.VISUAL_TEST_TYPE)),
     customize: Kaspresso.Builder.() -> Unit = {}
 ): Kaspresso.Builder = simple {
     if (!isAndroidRuntime) {
@@ -72,6 +83,7 @@ fun Kaspresso.Builder.Companion.withForcedAllureSupport(
     customize.invoke(this)
     val instrumentalDependencyProvider = instrumentalDependencyProviderFactory.getComponentProvider<Kaspresso>(instrumentation)
     forceAllureSupportFileProviders(instrumentalDependencyProvider)
+    initVisualTestParams(visualTestParams)
     addRunListenersIfNeeded(instrumentalDependencyProvider)
 }.apply {
     postInitAllure(shouldRecordVideo, builder = this)
@@ -98,14 +110,32 @@ private fun Kaspresso.Builder.forceAllureSupportFileProviders(provider: Instrume
     resourceFilesProvider = allureResourcesFilesProvider
 }
 
+private fun Kaspresso.Builder.initVisualTestParams(visualParams: VisualTestParams) {
+    visualTestParams = visualParams
+    testLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_TEST_LOGGER_TAG)
+    libLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_LIB_LOGGER_TAG)
+
+    screenshotsComparator = AllureScreenshotsComparator(
+        visualTestParams,
+        testLogger,
+        resourcesRootDirsProvider,
+        resourcesDirsProvider,
+        resourceFileNamesProvider,
+    )
+    adbServer = AdbServerImpl(LogLevel.WARN, libLogger)
+    files = FilesImpl(libLogger, adbServer)
+    visualTestWatcher = AllureVisualTestWatcher(visualTestParams, testLogger, (dirsProvider as AllureDirsProvider), resourcesRootDirsProvider, files)
+}
+
 private fun Kaspresso.Builder.addRunListenersIfNeeded(provider: InstrumentalDependencyProvider) {
     provider.runNotifier.apply {
         addUniqueListener(::AllureRunListener)
         addUniqueListener {
             AllureResultsHack(
                 uiDevice = provider.uiDevice,
                 resourcesRootDirsProvider = resourcesRootDirsProvider as DefaultAllureResourcesRootDirsProvider,
-                dirsProvider = dirsProvider as AllureDirsProvider
+                dirsProvider = dirsProvider as AllureDirsProvider,
+                visualTestParams = visualTestParams,
             )
         }
     }
@@ -126,6 +156,7 @@ private fun postInitAllure(shouldRecordVideo: Boolean, builder: Kaspresso.Builde
             DumpLogcatTestInterceptor(logcatDumper),
             ScreenshotTestInterceptor(screenshots),
             DumpViewsTestInterceptor(viewHierarchyDumper),
+            VisualTestLateFailInterceptor(),
         )
     )
     if (shouldRecordVideo) {

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt

@@ -0,0 +1,16 @@
+package com.kaspersky.components.alluresupport.interceptors.testrun
+
+import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag
+import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl
+import com.kaspersky.kaspresso.interceptors.watcher.testcase.TestRunWatcherInterceptor
+import com.kaspersky.kaspresso.testcases.models.info.TestInfo
+
+class VisualTestLateFailInterceptor : TestRunWatcherInterceptor {
+    override fun onAfterSectionStarted(testInfo: TestInfo) {
+        if (AllureVisualTestFlag.shouldFailLate.get()) {
+            // Wrap with assertion error so test would be marked as FAILED instead of BROKEN
+            // See https://github.com/allure-framework/allure-kotlin allure-kotlin-commons/src/main/kotlin/io/qameta/allure/kotlin/util/ResultsUtils.kt
+            throw AssertionError(ScreenshotsImpl.ScreenshotDoesntMatchException("There were failed screenshot comparisons. Check the allure report"))
+        }
+    }
+}

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureResultsHack.kt

@@ -6,14 +6,17 @@ import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
 import com.kaspersky.components.alluresupport.files.resources.AllureResourcesRootDirsProvider
 import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider
 import com.kaspersky.kaspresso.runner.listener.KaspressoRunListener
+import com.kaspersky.kaspresso.visual.VisualTestParams
+import com.kaspersky.kaspresso.visual.VisualTestType
 import io.qameta.allure.kotlin.Allure
 import org.junit.runner.Result
 import java.io.File
 
 class AllureResultsHack(
     private val uiDevice: UiDevice,
+    private val visualTestParams: VisualTestParams,
     resourcesRootDirsProvider: AllureResourcesRootDirsProvider,
-    dirsProvider: AllureDirsProvider,
+    private val dirsProvider: AllureDirsProvider,
 ) : KaspressoRunListener {
 
     private val allureResultsSourceDir: File =
@@ -48,6 +51,13 @@ class AllureResultsHack(
 
         allureResultsSourceDir.deleteRecursively()
         stubVideosDir.deleteRecursively()
+
+        if (visualTestParams.testType == VisualTestType.Record) {
+            val rootDir = dirsProvider.provideNew(File("")).absolutePath
+            val newScreenshotsDir = File(rootDir, File(visualTestParams.hostScreenshotsDir).name)
+            val targetScreenshotsDir = dirsProvider.provideNewOnSdCard(File(visualTestParams.hostScreenshotsDir))
+            newScreenshotsDir.copyRecursively(targetScreenshotsDir)
+        }
     }
 
     data class VideoBinding(

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureVisualTestFlag.kt

@@ -0,0 +1,11 @@
+package com.kaspersky.components.alluresupport.results
+
+import java.util.concurrent.atomic.AtomicBoolean
+
+// TODO(Nikita Evdokimov) - Certainly there should be a better way
+/**
+ * @see com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor
+ */
+object AllureVisualTestFlag {
+    val shouldFailLate = AtomicBoolean(false)
+}

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt

@@ -0,0 +1,35 @@
+package com.kaspersky.components.alluresupport.visual
+
+import android.graphics.Bitmap
+import com.kaspersky.components.alluresupport.files.attachScreenshotToAllureReport
+import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider
+import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider
+import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider
+import com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator
+import com.kaspersky.kaspresso.logger.Logger
+import com.kaspersky.kaspresso.visual.VisualTestParams
+import java.io.File
+
+class AllureScreenshotsComparator(
+    visualTestParams: VisualTestParams,
+    logger: Logger,
+    resourcesRootDirsProvider: ResourcesRootDirsProvider,
+    resourcesDirsProvider: ResourcesDirsProvider,
+    resourceFileNamesProvider: ResourceFileNamesProvider,
+) : DefaultScreenshotsComparator(visualTestParams, logger, resourcesRootDirsProvider, resourcesDirsProvider, resourceFileNamesProvider) {
+    override fun compare(originalScreenshot: File, newScreenshot: File): Boolean {
+        val doScreenshotsMatch = super.compare(originalScreenshot, newScreenshot)
+        if (!doScreenshotsMatch) {
+            originalScreenshot.attachScreenshotToAllureReport()
+            newScreenshot.attachScreenshotToAllureReport()
+        }
+
+        return doScreenshotsMatch
+    }
+
+    override fun processScreenshotDiff(original: Bitmap, diffPixels: IntArray, diffName: String): File {
+        return super.processScreenshotDiff(original, diffPixels, diffName).also {
+            it.attachScreenshotToAllureReport()
+        }
+    }
+}

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt

@@ -0,0 +1,35 @@
+package com.kaspersky.components.alluresupport.visual
+
+import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag
+import com.kaspersky.components.alluresupport.withForcedAllureSupport
+import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl
+import com.kaspersky.kaspresso.kaspresso.Kaspresso
+import com.kaspersky.kaspresso.testcases.api.testcase.VisualTestCase
+import io.qameta.allure.kotlin.Allure
+import io.qameta.allure.kotlin.model.Status
+import io.qameta.allure.kotlin.model.StatusDetails
+
+abstract class AllureVisualTestCase(
+    private val failEarly: Boolean = false,
+    kaspressoBuilder: Kaspresso.Builder = Kaspresso.Builder.withForcedAllureSupport()
+) : VisualTestCase(kaspressoBuilder = kaspressoBuilder) {
+
+    override fun assertScreenshot(tag: String, isFullWindow: Boolean) {
+        try {
+            device.screenshots.assert(tag, isFullWindow)
+        } catch (ex: ScreenshotsImpl.ScreenshotDoesntMatchException) {
+            if (failEarly) {
+                // Wrap with assertion error so test would be marked as FAILED instead of BROKEN
+                // See https://github.com/allure-framework/allure-kotlin allure-kotlin-commons/src/main/kotlin/io/qameta/allure/kotlin/util/ResultsUtils.kt
+                throw AssertionError(ex)
+            }
+
+            Allure.lifecycle.updateStep {
+                it.status = Status.FAILED
+                it.statusDetails = StatusDetails(known = true, muted = true, message = ex.message, trace = ex.stackTraceToString())
+            }
+            Allure.lifecycle.stopStep()
+            AllureVisualTestFlag.shouldFailLate.set(true)
+        }
+    }
+}

allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestWatcher.kt

@@ -0,0 +1,48 @@
+package com.kaspersky.components.alluresupport.visual
+
+import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider
+import com.kaspersky.kaspresso.device.files.Files
+import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider
+import com.kaspersky.kaspresso.logger.Logger
+import com.kaspersky.kaspresso.visual.VisualTestParams
+import com.kaspersky.kaspresso.visual.VisualTestType
+import com.kaspersky.kaspresso.visual.VisualTestWatcher
+import java.io.File
+
+class AllureVisualTestWatcher(
+    private val params: VisualTestParams,
+    private val logger: Logger,
+    private val dirsProvider: AllureDirsProvider,
+    resourcesRootDirsProvider: ResourcesRootDirsProvider,
+    private val files: Files,
+) : VisualTestWatcher {
+
+    private val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir)
+    private val originalScreenshotsTargetDir: File
+        get() {
+            val rootDir = dirsProvider.provideNewOnSdCard(File("")).absolutePath
+            return File(rootDir, File(params.hostScreenshotsDir).name)
+        }
+
+    override fun prepare() {
+        logger.i("Visual test run started. Parameters: $params")
+
+        if (params.testType == VisualTestType.Compare) {
+            logger.i("Pushing the screenshots unto the device...")
+            dirsProvider.provideCleared(diffDir)
+
+            // Allure stores all files in the app's private directory. We can't "adb push" directly there,
+            // so we have to do this in 2 steps
+            dirsProvider.provideCleared(originalScreenshotsTargetDir)
+            val tmp = dirsProvider.provideNewOnSdCard(File(""))
+            files.push(params.hostScreenshotsDir, tmp.absolutePath)
+            val target = dirsProvider.provideNew(File("")).resolve(params.hostScreenshotsDir)
+            File(tmp, params.hostScreenshotsDir).copyRecursively(target, overwrite = true)
+            logger.i("Done pushing the screenshots unto the device")
+        }
+    }
+
+    override fun cleanUp() {
+        // Do nothing
+    }
+}

build-logic/android/src/main/kotlin/convention.android-base.gradle.kts

@@ -32,5 +32,6 @@ configure<BaseExtension> {
         resValues = false
         shaders = false
         viewBinding = false
+        buildConfig = true
     }
 }

kaspresso/build.gradle.kts

@@ -7,6 +7,11 @@ plugins {
 
 android {
     namespace = "com.kaspersky.kaspresso"
+
+    defaultConfig {
+        buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE")
+            ?: findProperty("kaspresso.visualTestType")?.toString() ?: "\"Record\"") // [Record, Compare]
+    }
 }
 
 publish {

kaspresso/gradle.properties

@@ -1,4 +1,4 @@
 publish.artifactGroup=com.kaspersky.android-components
 publish.artifactName=kaspresso
 publish.publicationName=Kaspresso
-publish.bintrayRepo=Kaspresso
+publish.bintrayRepo=Kaspresso

kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/files/FilesImpl.kt

@@ -8,7 +8,7 @@ import com.kaspersky.kaspresso.logger.UiTestLogger
  */
 class FilesImpl(
     private val logger: UiTestLogger,
-    private val adbServer: AdbServer
+    private val adbServer: AdbServer,
 ) : Files {
 
     /**
@@ -20,7 +20,7 @@ class FilesImpl(
      * @param devicePath a path to copy.
      */
     override fun push(serverPath: String, devicePath: String) {
-        adbServer.performAdb("push $serverPath $devicePath")
+        adbServer.performAdb("push", listOf(serverPath, devicePath))
         logger.i("Push file from $serverPath to $devicePath")
     }
 
@@ -32,7 +32,7 @@ class FilesImpl(
      * @param path a path to remove
      */
     override fun remove(path: String) {
-        adbServer.performShell("rm -f $path")
+        adbServer.performShell("rm", listOf("-rf", path))
         logger.i("Remove file from $path")
     }
 
@@ -45,8 +45,8 @@ class FilesImpl(
      * @param serverPath a path to copy. (If empty - pulls in adbServer directory (folder with file "adbserver-desktop.jar"))
      */
     override fun pull(devicePath: String, serverPath: String) {
-        adbServer.performCmd("mkdir -p $serverPath")
-        adbServer.performAdb("pull $devicePath $serverPath")
+        adbServer.performCmd("mkdir", listOf("-p", serverPath))
+        adbServer.performAdb("pull", listOf(devicePath, serverPath))
         logger.i("Pull file from $devicePath to $serverPath")
     }
 }

kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/Screenshots.kt

@@ -42,4 +42,8 @@ interface Screenshots {
      * @param tag a unique tag to further identify the screenshot. Must match [a-zA-Z0-9_-]+.
      */
     fun takeFullWindowAndApply(tag: String, block: File.() -> Unit)
+
+    fun assert(tag: String, isFullWindow: Boolean)
+
+    fun assertAndApply(tag: String, isFullWindow: Boolean, block: File.() -> Unit)
 }