From 6065dd45c560948f2e1cc38a9e4d8f822f594f96 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Wed, 15 May 2024 14:11:15 +0300 Subject: [PATCH 01/10] TECH: Screenshot comparison tests --- allure-support/build.gradle.kts | 1 + .../AllureSupportKaspressoBuilder.kt | 28 ++++- .../testrun/VisualTestLateFailInterceptor.kt | 14 +++ .../results/AllureResultsHack.kt | 12 +- .../results/AllureVisualTestFlag.kt | 11 ++ .../visual/AllureScreenshotsComparator.kt | 33 ++++++ .../visual/AllureVisualTestCase.kt | 31 ++++++ .../visual/AllureVisualTestWatcher.kt | 48 ++++++++ .../kotlin/convention.android-base.gradle.kts | 1 + gradle.properties | 1 + gradle/libs.versions.toml | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- kaspresso/build.gradle.kts | 4 + kaspresso/gradle.properties | 3 +- .../kaspresso/device/files/FilesImpl.kt | 10 +- .../device/screenshots/Screenshots.kt | 4 + .../device/screenshots/ScreenshotsImpl.kt | 66 ++++++++++- .../resources/ResourcesRootDirsProvider.kt | 2 + .../impl/DefaultResourcesRootDirsProvider.kt | 2 + .../visual/DefaultScreenshotsComparator.kt | 104 ++++++++++++++++++ .../visual/DefaultVisualTestWatcher.kt | 52 +++++++++ .../kaspresso/kaspresso/Kaspresso.kt | 34 +++++- .../com/kaspersky/kaspresso/params/Params.kt | 5 +- .../api/testcase/DocLocScreenshotTestCase.kt | 2 + .../testcases/api/testcase/VisualTestCase.kt | 26 +++++ .../kaspresso/visual/ScreenshotsComparator.kt | 7 ++ .../kaspresso/visual/VisualTestParams.kt | 11 ++ .../kaspresso/visual/VisualTestType.kt | 5 + .../kaspresso/visual/VisualTestWatcher.kt | 6 + .../build.gradle.kts | 4 +- .../alluresupport/sample/AllureVisualTest.kt | 35 ++++++ .../src/main/AndroidManifest.xml | 1 + .../simple_tests/CustomizedSimpleTest.kt | 2 + .../kaspressample/visual/VisualTestSample.kt | 34 ++++++ 34 files changed, 581 insertions(+), 22 deletions(-) create mode 100644 allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt create mode 100644 allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureVisualTestFlag.kt create mode 100644 allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt create mode 100644 allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt create mode 100644 allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestWatcher.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/VisualTestCase.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/ScreenshotsComparator.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestType.kt create mode 100644 kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestWatcher.kt create mode 100644 samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt create mode 100644 samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/visual/VisualTestSample.kt diff --git a/allure-support/build.gradle.kts b/allure-support/build.gradle.kts index 7fb3ac15e..2eb3ce657 100644 --- a/allure-support/build.gradle.kts +++ b/allure-support/build.gradle.kts @@ -17,6 +17,7 @@ dependencies { api(libs.bundles.allure) implementation(projects.kaspresso) + implementation(projects.adbServer.adbServerCommon) implementation(libs.kotlinStdlib) implementation(libs.truth) diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt index 6ee0272cb..7c73fa4f0 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt @@ -1,5 +1,6 @@ package com.kaspersky.components.alluresupport +import com.kaspersky.adbserver.common.log.logger.LogLevel import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider import com.kaspersky.components.alluresupport.files.resources.impl.DefaultAllureResourcesRootDirsProvider @@ -10,8 +11,15 @@ import com.kaspersky.components.alluresupport.interceptors.testrun.DumpViewsTest import com.kaspersky.components.alluresupport.interceptors.testrun.HackyVideoRecordingTestInterceptor import com.kaspersky.components.alluresupport.interceptors.testrun.ScreenshotTestInterceptor import com.kaspersky.components.alluresupport.interceptors.testrun.VideoRecordingTestInterceptor +import com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor import com.kaspersky.components.alluresupport.results.AllureResultsHack import com.kaspersky.components.alluresupport.runlisteners.AllureRunListener +import com.kaspersky.components.alluresupport.visual.AllureScreenshotsComparator +import com.kaspersky.components.alluresupport.visual.AllureVisualTestWatcher +import com.kaspersky.kaspresso.BuildConfig +import com.kaspersky.kaspresso.device.Device +import com.kaspersky.kaspresso.device.files.FilesImpl +import com.kaspersky.kaspresso.device.server.AdbServerImpl import com.kaspersky.kaspresso.files.dirs.DefaultDirsProvider import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFileNamesProvider import com.kaspersky.kaspresso.files.resources.impl.DefaultResourceFilesProvider @@ -19,8 +27,11 @@ import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirNameProvi import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirsProvider import com.kaspersky.kaspresso.instrumental.InstrumentalDependencyProvider import com.kaspersky.kaspresso.kaspresso.Kaspresso +import com.kaspersky.kaspresso.logger.UiTestLoggerImpl import com.kaspersky.kaspresso.runner.listener.addUniqueListener import com.kaspersky.kaspresso.runner.listener.getUniqueListener +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType /** * Kaspresso Builder that includes all appropriate interceptors to support rich Allure reports. @@ -64,6 +75,7 @@ fun Kaspresso.Builder.addAllureSupport(): Kaspresso.Builder = apply { */ fun Kaspresso.Builder.Companion.withForcedAllureSupport( shouldRecordVideo: Boolean = true, + visualTestParams: VisualTestParams = VisualTestParams(testType = VisualTestType.valueOf(BuildConfig.VISUAL_TEST_TYPE)), customize: Kaspresso.Builder.() -> Unit = {} ): Kaspresso.Builder = simple { if (!isAndroidRuntime) { @@ -72,6 +84,7 @@ fun Kaspresso.Builder.Companion.withForcedAllureSupport( customize.invoke(this) val instrumentalDependencyProvider = instrumentalDependencyProviderFactory.getComponentProvider(instrumentation) forceAllureSupportFileProviders(instrumentalDependencyProvider) + initVisualTestParams(visualTestParams) addRunListenersIfNeeded(instrumentalDependencyProvider) }.apply { postInitAllure(shouldRecordVideo, builder = this) @@ -98,6 +111,17 @@ private fun Kaspresso.Builder.forceAllureSupportFileProviders(provider: Instrume resourceFilesProvider = allureResourcesFilesProvider } +private fun Kaspresso.Builder.initVisualTestParams(visualParams: VisualTestParams) { + visualTestParams = visualParams + testLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_TEST_LOGGER_TAG) + libLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_LIB_LOGGER_TAG) + + screenshotsComparator = AllureScreenshotsComparator(visualTestParams, testLogger, dirsProvider, resourcesRootDirsProvider) + adbServer = AdbServerImpl(LogLevel.WARN, libLogger) + files = FilesImpl(libLogger, adbServer) + visualTestWatcher = AllureVisualTestWatcher(visualTestParams, testLogger, (dirsProvider as AllureDirsProvider), resourcesRootDirsProvider, files) +} + private fun Kaspresso.Builder.addRunListenersIfNeeded(provider: InstrumentalDependencyProvider) { provider.runNotifier.apply { addUniqueListener(::AllureRunListener) @@ -105,7 +129,8 @@ private fun Kaspresso.Builder.addRunListenersIfNeeded(provider: InstrumentalDepe AllureResultsHack( uiDevice = provider.uiDevice, resourcesRootDirsProvider = resourcesRootDirsProvider as DefaultAllureResourcesRootDirsProvider, - dirsProvider = dirsProvider as AllureDirsProvider + dirsProvider = dirsProvider as AllureDirsProvider, + visualTestParams = visualTestParams, ) } } @@ -126,6 +151,7 @@ private fun postInitAllure(shouldRecordVideo: Boolean, builder: Kaspresso.Builde DumpLogcatTestInterceptor(logcatDumper), ScreenshotTestInterceptor(screenshots), DumpViewsTestInterceptor(viewHierarchyDumper), + VisualTestLateFailInterceptor(), ) ) if (shouldRecordVideo) { diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt new file mode 100644 index 000000000..023469630 --- /dev/null +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt @@ -0,0 +1,14 @@ +package com.kaspersky.components.alluresupport.interceptors.testrun + +import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag +import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl +import com.kaspersky.kaspresso.interceptors.watcher.testcase.TestRunWatcherInterceptor +import com.kaspersky.kaspresso.testcases.models.info.TestInfo + +class VisualTestLateFailInterceptor : TestRunWatcherInterceptor { + override fun onAfterSectionStarted(testInfo: TestInfo) { + if (AllureVisualTestFlag.shouldFailLate.get()) { + throw ScreenshotsImpl.ScreenshotDoesntMatchException("There were failed screenshot comparisons. Check the allure report") + } + } +} diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureResultsHack.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureResultsHack.kt index 216f5f0ec..75546d0e6 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureResultsHack.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureResultsHack.kt @@ -6,14 +6,17 @@ import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider import com.kaspersky.components.alluresupport.files.resources.AllureResourcesRootDirsProvider import com.kaspersky.components.alluresupport.files.resources.impl.AllureResourceFilesProvider import com.kaspersky.kaspresso.runner.listener.KaspressoRunListener +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType import io.qameta.allure.kotlin.Allure import org.junit.runner.Result import java.io.File class AllureResultsHack( private val uiDevice: UiDevice, + private val visualTestParams: VisualTestParams, resourcesRootDirsProvider: AllureResourcesRootDirsProvider, - dirsProvider: AllureDirsProvider, + private val dirsProvider: AllureDirsProvider, ) : KaspressoRunListener { private val allureResultsSourceDir: File = @@ -48,6 +51,13 @@ class AllureResultsHack( allureResultsSourceDir.deleteRecursively() stubVideosDir.deleteRecursively() + + if (visualTestParams.testType == VisualTestType.Record) { + val rootDir = dirsProvider.provideNew(File("")).absolutePath + val newScreenshotsDir = File(rootDir, File(visualTestParams.hostScreenshotsDir).name) + val targetScreenshotsDir = dirsProvider.provideNewOnSdCard(File(visualTestParams.hostScreenshotsDir)) + newScreenshotsDir.copyRecursively(targetScreenshotsDir) + } } data class VideoBinding( diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureVisualTestFlag.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureVisualTestFlag.kt new file mode 100644 index 000000000..f56973276 --- /dev/null +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/results/AllureVisualTestFlag.kt @@ -0,0 +1,11 @@ +package com.kaspersky.components.alluresupport.results + +import java.util.concurrent.atomic.AtomicBoolean + +// TODO(Nikita Evdokimov) - Certainly there should be a better way +/** + * @see com.kaspersky.components.alluresupport.interceptors.testrun.VisualTestLateFailInterceptor + */ +object AllureVisualTestFlag { + val shouldFailLate = AtomicBoolean(false) +} diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt new file mode 100644 index 000000000..5f66ac524 --- /dev/null +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt @@ -0,0 +1,33 @@ +package com.kaspersky.components.alluresupport.visual + +import android.graphics.Bitmap +import com.kaspersky.components.alluresupport.files.attachScreenshotToAllureReport +import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider +import com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator +import com.kaspersky.kaspresso.logger.Logger +import com.kaspersky.kaspresso.visual.VisualTestParams +import java.io.File + +class AllureScreenshotsComparator( + visualTestParams: VisualTestParams, + logger: Logger, + dirsProvider: DirsProvider, + resourcesRootDirsProvider: ResourcesRootDirsProvider, +) : DefaultScreenshotsComparator(visualTestParams, logger, dirsProvider, resourcesRootDirsProvider) { + override fun compare(originalScreenshot: File, newScreenshot: File): Boolean { + val doScreenshotsMatch = super.compare(originalScreenshot, newScreenshot) + if (!doScreenshotsMatch) { + originalScreenshot.attachScreenshotToAllureReport() + newScreenshot.attachScreenshotToAllureReport() + } + + return doScreenshotsMatch + } + + override fun processScreenshotDiff(original: Bitmap, diffPixels: IntArray, diffName: String): File { + return super.processScreenshotDiff(original, diffPixels, diffName).also { + it.attachScreenshotToAllureReport() + } + } +} diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt new file mode 100644 index 000000000..5f25e2f54 --- /dev/null +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt @@ -0,0 +1,31 @@ +package com.kaspersky.components.alluresupport.visual + +import com.kaspersky.components.alluresupport.results.AllureVisualTestFlag +import com.kaspersky.components.alluresupport.withForcedAllureSupport +import com.kaspersky.kaspresso.device.screenshots.ScreenshotsImpl +import com.kaspersky.kaspresso.kaspresso.Kaspresso +import com.kaspersky.kaspresso.testcases.api.testcase.VisualTestCase +import io.qameta.allure.kotlin.Allure +import io.qameta.allure.kotlin.model.Status +import io.qameta.allure.kotlin.model.StatusDetails + +abstract class AllureVisualTestCase( + private val failEarly: Boolean = false, + kaspressoBuilder: Kaspresso.Builder = Kaspresso.Builder.withForcedAllureSupport() +) : VisualTestCase(kaspressoBuilder = kaspressoBuilder) { + + override fun assertScreenshot(tag: String, isFullWindow: Boolean) { + try { + device.screenshots.assert(tag, isFullWindow) + } catch (ex: ScreenshotsImpl.ScreenshotDoesntMatchException) { + if (failEarly) { throw ex } + + Allure.lifecycle.updateStep { + it.status = Status.FAILED + it.statusDetails = StatusDetails(known = true, muted = true, message = ex.message, trace = ex.stackTraceToString()) + } + Allure.lifecycle.stopStep() + AllureVisualTestFlag.shouldFailLate.set(true) + } + } +} diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestWatcher.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestWatcher.kt new file mode 100644 index 000000000..c123f9b84 --- /dev/null +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestWatcher.kt @@ -0,0 +1,48 @@ +package com.kaspersky.components.alluresupport.visual + +import com.kaspersky.components.alluresupport.files.dirs.AllureDirsProvider +import com.kaspersky.kaspresso.device.files.Files +import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider +import com.kaspersky.kaspresso.logger.Logger +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType +import com.kaspersky.kaspresso.visual.VisualTestWatcher +import java.io.File + +class AllureVisualTestWatcher( + private val params: VisualTestParams, + private val logger: Logger, + private val dirsProvider: AllureDirsProvider, + resourcesRootDirsProvider: ResourcesRootDirsProvider, + private val files: Files, +) : VisualTestWatcher { + + private val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir) + private val originalScreenshotsTargetDir: File + get() { + val rootDir = dirsProvider.provideNewOnSdCard(File("")).absolutePath + return File(rootDir, File(params.hostScreenshotsDir).name) + } + + override fun prepare() { + logger.i("Visual test run started. Parameters: $params") + + if (params.testType == VisualTestType.Compare) { + logger.i("Pushing the screenshots unto the device...") + dirsProvider.provideCleared(diffDir) + + // Allure stores all files in the app's private directory. We can't "adb push" directly there, + // so we have to do this in 2 steps + dirsProvider.provideCleared(originalScreenshotsTargetDir) + val tmp = dirsProvider.provideNewOnSdCard(File("")) + files.push(params.hostScreenshotsDir, tmp.absolutePath) + val target = dirsProvider.provideNew(File("")).resolve(params.hostScreenshotsDir) + File(tmp, params.hostScreenshotsDir).copyRecursively(target, overwrite = true) + logger.i("Done pushing the screenshots unto the device") + } + } + + override fun cleanUp() { + // Do nothing + } +} diff --git a/build-logic/android/src/main/kotlin/convention.android-base.gradle.kts b/build-logic/android/src/main/kotlin/convention.android-base.gradle.kts index 3376cbd1f..6527cd6e6 100644 --- a/build-logic/android/src/main/kotlin/convention.android-base.gradle.kts +++ b/build-logic/android/src/main/kotlin/convention.android-base.gradle.kts @@ -32,5 +32,6 @@ configure { resValues = false shaders = false viewBinding = false + buildConfig = true } } diff --git a/gradle.properties b/gradle.properties index 737f17116..b779c4bd1 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,6 +4,7 @@ org.gradle.caching=true kotlin.code.style=official +android.injected.androidTest.leaveApksInstalledAfterRun=true android.useAndroidX = true kaspresso.version=1.5.4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 493f6b7bf..36600595f 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ thirdPartyReport = "0.19.1035" [libraries] # plugins kotlinPlugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } -androidPlugin = "com.android.tools.build:gradle:8.1.4" +androidPlugin = "com.android.tools.build:gradle:8.2.0" versionsPlugin = "com.github.ben-manes:gradle-versions-plugin:0.50.0" thirdPartyReportPlugin = { module = "com.kaspersky.gradle:third-party-report", version.ref = "thirdPartyReport" } airPlugin = { module = "com.kaspersky.gradle:air", version.ref = "thirdPartyReport" } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 59bc51a20..15de90249 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/kaspresso/build.gradle.kts b/kaspresso/build.gradle.kts index a52cfc7ec..679ec613a 100644 --- a/kaspresso/build.gradle.kts +++ b/kaspresso/build.gradle.kts @@ -7,6 +7,10 @@ plugins { android { namespace = "com.kaspersky.kaspresso" + + defaultConfig { + buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE") ?: property("kaspresso.visualTestType").toString()) // [Record, Compare] + } } publish { diff --git a/kaspresso/gradle.properties b/kaspresso/gradle.properties index 82b88875a..3b77fb9dd 100644 --- a/kaspresso/gradle.properties +++ b/kaspresso/gradle.properties @@ -1,4 +1,5 @@ publish.artifactGroup=com.kaspersky.android-components publish.artifactName=kaspresso publish.publicationName=Kaspresso -publish.bintrayRepo=Kaspresso \ No newline at end of file +publish.bintrayRepo=Kaspresso +kaspresso.visualTestType="Compare" diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/files/FilesImpl.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/files/FilesImpl.kt index 96f560e2d..7309a3175 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/files/FilesImpl.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/files/FilesImpl.kt @@ -8,7 +8,7 @@ import com.kaspersky.kaspresso.logger.UiTestLogger */ class FilesImpl( private val logger: UiTestLogger, - private val adbServer: AdbServer + private val adbServer: AdbServer, ) : Files { /** @@ -20,7 +20,7 @@ class FilesImpl( * @param devicePath a path to copy. */ override fun push(serverPath: String, devicePath: String) { - adbServer.performAdb("push $serverPath $devicePath") + adbServer.performAdb("push", listOf(serverPath, devicePath)) logger.i("Push file from $serverPath to $devicePath") } @@ -32,7 +32,7 @@ class FilesImpl( * @param path a path to remove */ override fun remove(path: String) { - adbServer.performShell("rm -f $path") + adbServer.performShell("rm", listOf("-rf", path)) logger.i("Remove file from $path") } @@ -45,8 +45,8 @@ class FilesImpl( * @param serverPath a path to copy. (If empty - pulls in adbServer directory (folder with file "adbserver-desktop.jar")) */ override fun pull(devicePath: String, serverPath: String) { - adbServer.performCmd("mkdir -p $serverPath") - adbServer.performAdb("pull $devicePath $serverPath") + adbServer.performCmd("mkdir", listOf("-p", serverPath)) + adbServer.performAdb("pull", listOf(devicePath, serverPath)) logger.i("Pull file from $devicePath to $serverPath") } } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/Screenshots.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/Screenshots.kt index 92b851bbd..a8cc5478c 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/Screenshots.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/Screenshots.kt @@ -42,4 +42,8 @@ interface Screenshots { * @param tag a unique tag to further identify the screenshot. Must match [a-zA-Z0-9_-]+. */ fun takeFullWindowAndApply(tag: String, block: File.() -> Unit) + + fun assert(tag: String, isFullWindow: Boolean) + + fun assertAndApply(tag: String, isFullWindow: Boolean, block: File.() -> Unit) } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt index 9926b69b7..d647b945b 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt @@ -2,8 +2,14 @@ package com.kaspersky.kaspresso.device.screenshots import android.util.Log import com.kaspersky.kaspresso.device.screenshots.screenshotmaker.ScreenshotMaker +import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.extensions.FileExtension +import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider import com.kaspersky.kaspresso.files.resources.ResourceFilesProvider import com.kaspersky.kaspresso.logger.UiTestLogger +import com.kaspersky.kaspresso.visual.ScreenshotsComparator +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType import java.io.File /** @@ -13,8 +19,18 @@ class ScreenshotsImpl( private val logger: UiTestLogger, private val resourceFilesProvider: ResourceFilesProvider, private val screenshotMaker: ScreenshotMaker, + private val screenshotsComparator: ScreenshotsComparator, + private val visualTestParams: VisualTestParams, + private val dirsProvider: DirsProvider, + private val resourceFileNamesProvider: ResourceFileNamesProvider, ) : Screenshots { + private val originalScreenshotsDir: File + get() { + val rootDir = dirsProvider.provideNew(File("")).absolutePath + return File(rootDir, File(visualTestParams.hostScreenshotsDir).name) + } + /** * Takes a screenshot if it is possible, otherwise logs the error. * By default a screenshot name looks like /screenshotRootDir////[tag].png @@ -24,17 +40,53 @@ class ScreenshotsImpl( * * @param tag a unique tag to further identify the screenshot. Must match [a-zA-Z0-9_-]+. */ - override fun take(tag: String): Unit = doTakeAndApply(tag, false, null) + override fun take(tag: String): Unit = doTakeAndApply(tag, isFull = false, block = null) + + override fun takeFullWindow(tag: String) = doTakeAndApply(tag, isFull = true, block = null) + + override fun takeAndApply(tag: String, block: File.() -> Unit): Unit = doTakeAndApply(tag, isFull = false, block = block) - override fun takeFullWindow(tag: String) = doTakeAndApply(tag, true, null) + override fun takeFullWindowAndApply(tag: String, block: File.() -> Unit) = doTakeAndApply(tag, isFull = true, block = block) - override fun takeAndApply(tag: String, block: File.() -> Unit): Unit = doTakeAndApply(tag, false, block) + override fun assert(tag: String, isFullWindow: Boolean) = assertImpl(tag, isFullWindow, block = null) - override fun takeFullWindowAndApply(tag: String, block: File.() -> Unit) = doTakeAndApply(tag, true, block) + override fun assertAndApply(tag: String, isFullWindow: Boolean, block: File.() -> Unit) = assertImpl(tag, isFullWindow, block) + + private fun assertImpl(tag: String, isFullWindow: Boolean, block: (File.() -> Unit)?) { + logger.i("Assert screenshot with tag: $tag") + lateinit var screenshot: File + val targetPath = if (visualTestParams.testType == VisualTestType.Compare) { + null + } else { + originalScreenshotsDir.mkdirs() + File(originalScreenshotsDir, resourceFileNamesProvider.getFileName(tag, FileExtension.PNG.toString())) + } + doTakeAndApply(tag = tag, isFull = isFullWindow, targetPath) { screenshot = this } + + if (visualTestParams.testType == VisualTestType.Compare) { + screenshot.compare() + } - private fun doTakeAndApply(tag: String, isFull: Boolean, block: (File.() -> Unit)?) { + block?.invoke(screenshot) + } + + private fun File.compare() { + logger.i("Comparing screenshot ${this.absolutePath}") + val originalScreenshot = File(originalScreenshotsDir, name) + assert(originalScreenshot.exists()) { + "Tried to assert screenshot $absolutePath, but failed to find matching " + + "original screenshot by the path: ${originalScreenshotsDir.absolutePath}/$name" + } + val doesMatch = screenshotsComparator.compare(originalScreenshot, this) + logger.i("Does screenshot $name matches the original one: $doesMatch") + if (!doesMatch) { + throw ScreenshotDoesntMatchException("Screenshot $name doesn't match the original one") + } + } + + private fun doTakeAndApply(tag: String, isFull: Boolean, targetPath: File? = null, block: (File.() -> Unit)?) { try { - val screenshotFile: File = resourceFilesProvider.provideScreenshotFile(tag) + val screenshotFile = targetPath ?: resourceFilesProvider.provideScreenshotFile(tag) if (isFull) { screenshotMaker.takeFullWindowScreenshot(screenshotFile) } else { @@ -46,4 +98,6 @@ class ScreenshotsImpl( logger.e("An error while making screenshot occurred: ${Log.getStackTraceString(e)}") } } + + class ScreenshotDoesntMatchException(message: String) : Exception(message) } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesRootDirsProvider.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesRootDirsProvider.kt index 5ac81ec06..afb8957ab 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesRootDirsProvider.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesRootDirsProvider.kt @@ -8,6 +8,8 @@ import java.io.File interface ResourcesRootDirsProvider { val logcatRootDir: File val screenshotsRootDir: File + val originalScreenshotsRootDir: File + val screenshotsDiffRootDir: File val videoRootDir: File val viewHierarchy: File } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt index fdab929f1..e965542e6 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt @@ -6,6 +6,8 @@ import java.io.File class DefaultResourcesRootDirsProvider : ResourcesRootDirsProvider { override val logcatRootDir = File("logcat") override val screenshotsRootDir = File("screenshots") + override val originalScreenshotsRootDir = File("original_screenshots") + override val screenshotsDiffRootDir = File("diff") override val videoRootDir = File("video") override val viewHierarchy = File("view_hierarchy") } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt new file mode 100644 index 000000000..e7fcb5642 --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt @@ -0,0 +1,104 @@ +package com.kaspersky.kaspresso.internal.visual + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Color +import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider +import com.kaspersky.kaspresso.logger.Logger +import com.kaspersky.kaspresso.visual.ScreenshotsComparator +import com.kaspersky.kaspresso.visual.VisualTestParams +import java.io.File +import java.io.FileOutputStream +import kotlin.math.abs + +open class DefaultScreenshotsComparator( + private val visualTestParams: VisualTestParams, + private val logger: Logger, + private val dirsProvider: DirsProvider, + private val resourcesRootDirsProvider: ResourcesRootDirsProvider, +) : ScreenshotsComparator { + + @Suppress("MagicNumber") + override fun compare(originalScreenshot: File, newScreenshot: File): Boolean { + val decodeOptions = BitmapFactory.Options().apply { inMutable = true } + val screenshot = BitmapFactory.decodeFile(newScreenshot.absolutePath, decodeOptions) + val original = BitmapFactory.decodeFile(originalScreenshot.absolutePath, decodeOptions) + + if (original.sameAs(screenshot)) { + return true + } + + val width: Int = original.width + val height: Int = original.height + val pixelsCount = width * height + val screenshotPixels = IntArray(pixelsCount) + val originalPixels = IntArray(pixelsCount) + val diffPixels = IntArray(pixelsCount) + + screenshot.getPixels(screenshotPixels, 0, width, 0, 0, width, height) + original.getPixels(originalPixels, 0, width, 0, 0, width, height) + + var totalDelta = 0 + for (pixelIndex in 0 until pixelsCount) { + val areColorsCorrect = checkColors(screenshotPixels[pixelIndex], originalPixels[pixelIndex]) + if (areColorsCorrect) { + diffPixels[pixelIndex] = Color.BLACK + } else { + totalDelta++ + diffPixels[pixelIndex] = Color.WHITE + } + } + + val diffValue = totalDelta * 100.0f / (width * height) + logger.i("${originalScreenshot.absolutePath} diff is $diffValue") + if (diffValue > visualTestParams.tolerance) { + val name = originalScreenshot.name + processScreenshotDiff(original, diffPixels, "diff_$name") + return false + } + + return true + } + + private fun checkColors(rgb1: Int, rgb2: Int): Boolean { + val colorTolerance = visualTestParams.colorTolerance + val r1 = Color.red(rgb1) + val g1 = Color.green(rgb1) + val b1 = Color.blue(rgb1) + val r2 = Color.red(rgb2) + val g2 = Color.green(rgb2) + val b2 = Color.blue(rgb2) + return abs(r1 - r2) <= colorTolerance && + abs(g1 - g2) <= colorTolerance && + abs(b1 - b2) <= colorTolerance + } + + protected open fun processScreenshotDiff(original: Bitmap, diffPixels: IntArray, diffName: String): File { + val width = original.width + val height = original.height + val diffBitmap = Bitmap.createBitmap(width, height, original.config) + diffBitmap.setPixels(diffPixels, 0, width, 0, 0, width, height) + val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir) + val screenshotDiff = File(diffDir, diffName) + val scaledBitmap = Bitmap.createScaledBitmap( + diffBitmap, + width, + height, + false, + ) + assert( + scaledBitmap.compress( + Bitmap.CompressFormat.PNG, + QUALITY, + FileOutputStream(screenshotDiff), + ) + ) + + return screenshotDiff + } + + companion object { + private const val QUALITY = 100 + } +} diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt new file mode 100644 index 000000000..f71c1752a --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt @@ -0,0 +1,52 @@ +package com.kaspersky.kaspresso.internal.visual + +import com.kaspersky.kaspresso.device.files.Files +import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider +import com.kaspersky.kaspresso.logger.Logger +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType +import com.kaspersky.kaspresso.visual.VisualTestWatcher +import java.io.File + +internal class DefaultVisualTestWatcher( + private val params: VisualTestParams, + private val logger: Logger, + private val dirsProvider: DirsProvider, + resourcesRootDirsProvider: ResourcesRootDirsProvider, + private val files: Files, +) : VisualTestWatcher { + private val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir) + private val originalScreenshotsTargetDir: File + get() { + val rootDir = dirsProvider.provideNew(File("")).absolutePath + return File(rootDir, File(params.hostScreenshotsDir).name) + } + private val newScreenshotsDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsRootDir) + + override fun prepare() { + logger.i("Visual test run started. Parameters: $params") + + if (params.testType == VisualTestType.Compare) { + logger.i("Pushing the screenshots unto the device...") + dirsProvider.provideCleared(diffDir) + + dirsProvider.provideCleared(originalScreenshotsTargetDir) + files.push(params.hostScreenshotsDir, dirsProvider.provideNew(File("")).absolutePath) + logger.i("Done pushing the screenshots unto the device") + } + } + + override fun cleanUp() { + logger.i("Visual test finished") + + if (params.testType == VisualTestType.Compare) { + logger.i("Pulling diff files from the device...") + files.pull(diffDir.absolutePath, ".") + logger.i("Done pulling diff files from the device") + logger.i("Pulling new screenshot files from the device...") + files.pull(newScreenshotsDir.absolutePath, ".") + logger.i("Done pulling new screenshot files from the device...") + } + } +} diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt index 710422f92..a1b4a56bc 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt @@ -9,6 +9,7 @@ import com.kaspersky.adbserver.common.log.logger.LogLevel import com.kaspersky.components.kautomator.KautomatorConfigurator import com.kaspersky.components.kautomator.intercept.interaction.UiDeviceInteraction import com.kaspersky.components.kautomator.intercept.interaction.UiObjectInteraction +import com.kaspersky.kaspresso.BuildConfig import com.kaspersky.kaspresso.device.Device import com.kaspersky.kaspresso.device.accessibility.Accessibility import com.kaspersky.kaspresso.device.accessibility.AccessibilityImpl @@ -112,6 +113,8 @@ import com.kaspersky.kaspresso.interceptors.watcher.view.impl.logging.LoggingVie import com.kaspersky.kaspresso.interceptors.watcher.view.impl.logging.LoggingViewAssertionWatcherInterceptor import com.kaspersky.kaspresso.interceptors.watcher.view.impl.logging.LoggingWebAssertionWatcherInterceptor import com.kaspersky.kaspresso.internal.runlisteners.artifactspull.ArtifactsPullRunListener +import com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator +import com.kaspersky.kaspresso.internal.visual.DefaultVisualTestWatcher import com.kaspersky.kaspresso.logger.UiTestLogger import com.kaspersky.kaspresso.logger.UiTestLoggerImpl import com.kaspersky.kaspresso.params.ArtifactsPullParams @@ -127,6 +130,10 @@ import com.kaspersky.kaspresso.params.SystemDialogsSafetyParams import com.kaspersky.kaspresso.params.VideoParams import com.kaspersky.kaspresso.runner.listener.addUniqueListener import com.kaspersky.kaspresso.testcases.core.testcontext.BaseTestContext +import com.kaspersky.kaspresso.visual.ScreenshotsComparator +import com.kaspersky.kaspresso.visual.VisualTestParams +import com.kaspersky.kaspresso.visual.VisualTestType +import com.kaspersky.kaspresso.visual.VisualTestWatcher import io.github.kakaocup.kakao.Kakao /** @@ -152,7 +159,8 @@ data class Kaspresso( internal val deviceBehaviorInterceptors: List, internal val stepWatcherInterceptors: List, internal val testRunWatcherInterceptors: List, - internal val resourceFilesProvider: ResourceFilesProvider + internal val resourceFilesProvider: ResourceFilesProvider, + internal val visualTestWatcher: VisualTestWatcher, ) { companion object { @@ -491,6 +499,17 @@ data class Kaspresso( * If it was not specified, the default implementation is used. */ lateinit var artifactsPullParams: ArtifactsPullParams + + /** + * Holds the [VisualTestParams]. + * If it was not specified, the default implementation is used. + */ + lateinit var visualTestParams: VisualTestParams + + lateinit var screenshotsComparator: ScreenshotsComparator + + lateinit var visualTestWatcher: VisualTestWatcher + /** * Holds an implementation of [DirsProvider] interface. If it was not specified, the default implementation is used. */ @@ -755,6 +774,9 @@ data class Kaspresso( if (!::elementLoaderParams.isInitialized) elementLoaderParams = ElementLoaderParams() if (!::clickParams.isInitialized) clickParams = ClickParams.default() if (!::artifactsPullParams.isInitialized) artifactsPullParams = ArtifactsPullParams(enabled = false) + if (!::visualTestParams.isInitialized) visualTestParams = VisualTestParams(testType = VisualTestType.valueOf(BuildConfig.VISUAL_TEST_TYPE)) + if (!::screenshotsComparator.isInitialized) screenshotsComparator = DefaultScreenshotsComparator(visualTestParams, testLogger, dirsProvider, resourcesRootDirsProvider) + if (!::visualTestWatcher.isInitialized) visualTestWatcher = DefaultVisualTestWatcher(visualTestParams, libLogger, dirsProvider, resourcesRootDirsProvider, files) if (!::screenshots.isInitialized) { screenshots = ScreenshotsImpl( @@ -766,7 +788,11 @@ data class Kaspresso( instrumentalDependencyProviderFactory.getComponentProvider(instrumentation), screenshotParams ) - ) + ), + visualTestParams = visualTestParams, + screenshotsComparator = screenshotsComparator, + dirsProvider = dirsProvider, + resourceFileNamesProvider = resourceFileNamesProvider, ) } @@ -977,7 +1003,8 @@ data class Kaspresso( videoParams = videoParams, elementLoaderParams = elementLoaderParams, systemDialogsSafetyParams = systemDialogsSafetyParams, - clickParams = clickParams + clickParams = clickParams, + visualTestParams = visualTestParams, ), viewActionWatcherInterceptors = viewActionWatcherInterceptors, @@ -997,6 +1024,7 @@ data class Kaspresso( stepWatcherInterceptors = stepWatcherInterceptors, testRunWatcherInterceptors = testRunWatcherInterceptors, + visualTestWatcher = visualTestWatcher, ) configurator.waitForIdleTimeout = kautomatorWaitForIdleSettings.waitForIdleTimeout diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/params/Params.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/params/Params.kt index c9d9eb6e9..40c64c3ac 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/params/Params.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/params/Params.kt @@ -1,5 +1,7 @@ package com.kaspersky.kaspresso.params +import com.kaspersky.kaspresso.visual.VisualTestParams + /** * The facade class for all Kaspresso parameters. */ @@ -12,5 +14,6 @@ data class Params( val videoParams: VideoParams, val elementLoaderParams: ElementLoaderParams, val systemDialogsSafetyParams: SystemDialogsSafetyParams, - val clickParams: ClickParams + val clickParams: ClickParams, + val visualTestParams: VisualTestParams, ) diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt index 2d1a75c88..b64357744 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt @@ -99,6 +99,8 @@ abstract class DocLocScreenshotTestCase( resourcesRootDirsProvider = object : ResourcesRootDirsProvider { override val logcatRootDir: File = File("logcat") override val screenshotsRootDir = screenshotsDirectory + override val originalScreenshotsRootDir = File("original_screenshots") + override val screenshotsDiffRootDir: File = File("diff") override val videoRootDir: File = File("video") override val viewHierarchy: File = File("view_hierarchy") }, diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/VisualTestCase.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/VisualTestCase.kt new file mode 100644 index 000000000..5bd1ddfce --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/VisualTestCase.kt @@ -0,0 +1,26 @@ +package com.kaspersky.kaspresso.testcases.api.testcase + +import com.kaspersky.kaspresso.kaspresso.Kaspresso +import com.kaspersky.kaspresso.testcases.core.testcontext.BaseTestContext +import com.kaspersky.kaspresso.testcases.core.testcontext.TestContext + +abstract class VisualTestCase( + kaspressoBuilder: Kaspresso.Builder = Kaspresso.Builder.simple(), +) : TestCase(kaspressoBuilder) { + + open fun runScreenshotTest( + before: (BaseTestContext.() -> Unit)? = null, + after: (BaseTestContext.() -> Unit)? = null, + test: TestContext.() -> Unit, + ) = before { + kaspresso.visualTestWatcher.prepare() + before?.invoke(this) + }.after { + kaspresso.visualTestWatcher.cleanUp() + after?.invoke(this) + }.run(test) + + open fun assertScreenshot(tag: String, isFullWindow: Boolean = false) { + device.screenshots.assert(tag, isFullWindow) + } +} diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/ScreenshotsComparator.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/ScreenshotsComparator.kt new file mode 100644 index 000000000..53e775853 --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/ScreenshotsComparator.kt @@ -0,0 +1,7 @@ +package com.kaspersky.kaspresso.visual + +import java.io.File + +interface ScreenshotsComparator { + fun compare(originalScreenshot: File, newScreenshot: File): Boolean +} diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt new file mode 100644 index 000000000..9aa67dfe9 --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt @@ -0,0 +1,11 @@ +package com.kaspersky.kaspresso.visual + +import android.annotation.SuppressLint + +@SuppressLint("SdCardPath") +data class VisualTestParams( + val testType: VisualTestType = VisualTestType.Record, + val hostScreenshotsDir: String = "original_screenshots", + val colorTolerance: Int = 1, + val tolerance: Float = 0.3f, +) diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestType.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestType.kt new file mode 100644 index 000000000..3da2e5b7d --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestType.kt @@ -0,0 +1,5 @@ +package com.kaspersky.kaspresso.visual + +enum class VisualTestType { + Record, Compare +} diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestWatcher.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestWatcher.kt new file mode 100644 index 000000000..39cb6e750 --- /dev/null +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestWatcher.kt @@ -0,0 +1,6 @@ +package com.kaspersky.kaspresso.visual + +interface VisualTestWatcher { + fun prepare() + fun cleanUp() +} diff --git a/samples/kaspresso-allure-support-sample/build.gradle.kts b/samples/kaspresso-allure-support-sample/build.gradle.kts index 7c39238d2..048c43b15 100644 --- a/samples/kaspresso-allure-support-sample/build.gradle.kts +++ b/samples/kaspresso-allure-support-sample/build.gradle.kts @@ -7,11 +7,11 @@ android { defaultConfig { applicationId = "com.kaspersky.kaspresso.alluresupport.sample" testInstrumentationRunner = "com.kaspersky.kaspresso.runner.KaspressoRunner" - testInstrumentationRunnerArguments["clearPackageData"] = "true" + testInstrumentationRunnerArguments["clearPackageData"] = "false" } testOptions { - execution = "ANDROIDX_TEST_ORCHESTRATOR" +// execution = "ANDROIDX_TEST_ORCHESTRATOR" } } diff --git a/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt b/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt new file mode 100644 index 000000000..2985df03f --- /dev/null +++ b/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt @@ -0,0 +1,35 @@ +package com.kaspersky.kaspresso.alluresupport.sample + +import android.Manifest +import androidx.test.ext.junit.rules.activityScenarioRule +import androidx.test.rule.GrantPermissionRule +import com.kaspersky.components.alluresupport.visual.AllureVisualTestCase +import com.kaspersky.kaspresso.alluresupport.sample.screen.MainScreen +import org.junit.Rule +import org.junit.Test + +class AllureVisualTest : AllureVisualTestCase(failEarly = false) { + @get:Rule + val activityRule = activityScenarioRule() + + @get:Rule + val runtimePermissionRule: GrantPermissionRule = GrantPermissionRule.grant( + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE + ) + + @Test + fun counter() = runScreenshotTest { + step("Launch the app") { + + MainScreen { + incrementButton.isDisplayed() + decrementButton.isDisplayed() + clearButton.isDisplayed() + valueText.isDisplayed() + } + + assertScreenshot("some_tag") + } + } +} diff --git a/samples/kaspresso-allure-support-sample/src/main/AndroidManifest.xml b/samples/kaspresso-allure-support-sample/src/main/AndroidManifest.xml index 3d542e133..8caf983ce 100644 --- a/samples/kaspresso-allure-support-sample/src/main/AndroidManifest.xml +++ b/samples/kaspresso-allure-support-sample/src/main/AndroidManifest.xml @@ -6,6 +6,7 @@ + () + + @Test + fun test() = runScreenshotTest { + step("Open Simple Screen") { + MainScreen { + simpleButton { + isVisible() + click() + assertScreenshot("some_tag") + } + } + } + } +} From 708689f96c4b55d700a08c21f4d42ec31a771124 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Fri, 16 Aug 2024 10:43:31 +0300 Subject: [PATCH 02/10] TECH: lint --- .../components/alluresupport/AllureSupportKaspressoBuilder.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt index 7c73fa4f0..55c2e4b3e 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt @@ -17,7 +17,6 @@ import com.kaspersky.components.alluresupport.runlisteners.AllureRunListener import com.kaspersky.components.alluresupport.visual.AllureScreenshotsComparator import com.kaspersky.components.alluresupport.visual.AllureVisualTestWatcher import com.kaspersky.kaspresso.BuildConfig -import com.kaspersky.kaspresso.device.Device import com.kaspersky.kaspresso.device.files.FilesImpl import com.kaspersky.kaspresso.device.server.AdbServerImpl import com.kaspersky.kaspresso.files.dirs.DefaultDirsProvider From c62ce39f6da7c40961977fad7717f1c2201d6598 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Tue, 5 Nov 2024 18:21:13 +0300 Subject: [PATCH 03/10] TECH: PR comments --- gradle.properties | 1 - gradle/libs.versions.toml | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- .../impl/DefaultResourcesRootDirsProvider.kt | 2 +- .../api/testcase/DocLocScreenshotTestCase.kt | 2 +- .../build.gradle.kts | 4 ++-- .../alluresupport/sample/AllureVisualTest.kt | 15 ++++++++------- .../simple_tests/CustomizedSimpleTest.kt | 2 +- .../kaspressample/visual/VisualTestSample.kt | 3 ++- 9 files changed, 17 insertions(+), 16 deletions(-) diff --git a/gradle.properties b/gradle.properties index b779c4bd1..737f17116 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,7 +4,6 @@ org.gradle.caching=true kotlin.code.style=official -android.injected.androidTest.leaveApksInstalledAfterRun=true android.useAndroidX = true kaspresso.version=1.5.4 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index ff3643e73..45e4fc994 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -17,7 +17,7 @@ thirdPartyReport = "0.19.1035" [libraries] # plugins kotlinPlugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } -androidPlugin = "com.android.tools.build:gradle:8.2.0" +androidPlugin = "com.android.tools.build:gradle:8.1.4" versionsPlugin = "com.github.ben-manes:gradle-versions-plugin:0.50.0" thirdPartyReportPlugin = { module = "com.kaspersky.gradle:third-party-report", version.ref = "thirdPartyReport" } airPlugin = { module = "com.kaspersky.gradle:air", version.ref = "thirdPartyReport" } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 15de90249..59bc51a20 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt index e965542e6..b5281a001 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesRootDirsProvider.kt @@ -7,7 +7,7 @@ class DefaultResourcesRootDirsProvider : ResourcesRootDirsProvider { override val logcatRootDir = File("logcat") override val screenshotsRootDir = File("screenshots") override val originalScreenshotsRootDir = File("original_screenshots") - override val screenshotsDiffRootDir = File("diff") + override val screenshotsDiffRootDir = File("screenshot_diffs") override val videoRootDir = File("video") override val viewHierarchy = File("view_hierarchy") } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt index b64357744..7a2e4a311 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/testcases/api/testcase/DocLocScreenshotTestCase.kt @@ -100,7 +100,7 @@ abstract class DocLocScreenshotTestCase( override val logcatRootDir: File = File("logcat") override val screenshotsRootDir = screenshotsDirectory override val originalScreenshotsRootDir = File("original_screenshots") - override val screenshotsDiffRootDir: File = File("diff") + override val screenshotsDiffRootDir: File = File("screenshot_diffs") override val videoRootDir: File = File("video") override val viewHierarchy: File = File("view_hierarchy") }, diff --git a/samples/kaspresso-allure-support-sample/build.gradle.kts b/samples/kaspresso-allure-support-sample/build.gradle.kts index 048c43b15..7c39238d2 100644 --- a/samples/kaspresso-allure-support-sample/build.gradle.kts +++ b/samples/kaspresso-allure-support-sample/build.gradle.kts @@ -7,11 +7,11 @@ android { defaultConfig { applicationId = "com.kaspersky.kaspresso.alluresupport.sample" testInstrumentationRunner = "com.kaspersky.kaspresso.runner.KaspressoRunner" - testInstrumentationRunnerArguments["clearPackageData"] = "false" + testInstrumentationRunnerArguments["clearPackageData"] = "true" } testOptions { -// execution = "ANDROIDX_TEST_ORCHESTRATOR" + execution = "ANDROIDX_TEST_ORCHESTRATOR" } } diff --git a/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt b/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt index 2985df03f..41339aabc 100644 --- a/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt +++ b/samples/kaspresso-allure-support-sample/src/androidTest/kotlin/com/kaspersky/kaspresso/alluresupport/sample/AllureVisualTest.kt @@ -20,16 +20,17 @@ class AllureVisualTest : AllureVisualTestCase(failEarly = false) { @Test fun counter() = runScreenshotTest { - step("Launch the app") { + step("Assert screenshot") { + assertScreenshot("some_tag") + } + step("Usual checks") { MainScreen { - incrementButton.isDisplayed() - decrementButton.isDisplayed() - clearButton.isDisplayed() - valueText.isDisplayed() + incrementButton { isDisplayed() } + decrementButton { isDisplayed() } + clearButton { isDisplayed() } + valueText { isDisplayed() } } - - assertScreenshot("some_tag") } } } diff --git a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt index 1f9349b8d..238c471c6 100644 --- a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt +++ b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt @@ -45,7 +45,7 @@ class CustomizedSimpleTest : TestCase( override val logcatRootDir = File("custom_logcat") override val screenshotsRootDir = File("custom_screenshots") override val originalScreenshotsRootDir = File("custom_original_screenshots") - override val screenshotsDiffRootDir = File("custom_diff") + override val screenshotsDiffRootDir = File("custom_screenshot_diffs") override val videoRootDir = File("custom_video") override val viewHierarchy = File("custom_view_hierarchy") } diff --git a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/visual/VisualTestSample.kt b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/visual/VisualTestSample.kt index a86cd1462..de5117910 100644 --- a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/visual/VisualTestSample.kt +++ b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/visual/VisualTestSample.kt @@ -13,7 +13,8 @@ class VisualTestSample : VisualTestCase() { @get:Rule val runtimePermissionRule: GrantPermissionRule = GrantPermissionRule.grant( Manifest.permission.WRITE_EXTERNAL_STORAGE, - Manifest.permission.READ_EXTERNAL_STORAGE + Manifest.permission.READ_EXTERNAL_STORAGE, + Manifest.permission.READ_MEDIA_IMAGES, ) @get:Rule From a596495f011fa2666ba5bb825026f2f68b099e9d Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Tue, 5 Nov 2024 18:21:59 +0300 Subject: [PATCH 04/10] TECH: Mark failed allure screenshot assertions as FAILED instead of BROKEN --- .../interceptors/testrun/VisualTestLateFailInterceptor.kt | 4 +++- .../components/alluresupport/visual/AllureVisualTestCase.kt | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt index 023469630..df1b74f25 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/interceptors/testrun/VisualTestLateFailInterceptor.kt @@ -8,7 +8,9 @@ import com.kaspersky.kaspresso.testcases.models.info.TestInfo class VisualTestLateFailInterceptor : TestRunWatcherInterceptor { override fun onAfterSectionStarted(testInfo: TestInfo) { if (AllureVisualTestFlag.shouldFailLate.get()) { - throw ScreenshotsImpl.ScreenshotDoesntMatchException("There were failed screenshot comparisons. Check the allure report") + // Wrap with assertion error so test would be marked as FAILED instead of BROKEN + // See https://github.com/allure-framework/allure-kotlin allure-kotlin-commons/src/main/kotlin/io/qameta/allure/kotlin/util/ResultsUtils.kt + throw AssertionError(ScreenshotsImpl.ScreenshotDoesntMatchException("There were failed screenshot comparisons. Check the allure report")) } } } diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt index 5f25e2f54..5930d0449 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureVisualTestCase.kt @@ -18,7 +18,11 @@ abstract class AllureVisualTestCase( try { device.screenshots.assert(tag, isFullWindow) } catch (ex: ScreenshotsImpl.ScreenshotDoesntMatchException) { - if (failEarly) { throw ex } + if (failEarly) { + // Wrap with assertion error so test would be marked as FAILED instead of BROKEN + // See https://github.com/allure-framework/allure-kotlin allure-kotlin-commons/src/main/kotlin/io/qameta/allure/kotlin/util/ResultsUtils.kt + throw AssertionError(ex) + } Allure.lifecycle.updateStep { it.status = Status.FAILED From 7728629a64a5eec88d3b8c09f920c7ba3ebb59ad Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Mon, 18 Nov 2024 10:59:28 +0300 Subject: [PATCH 05/10] TECH: Fix destination path; add documentation --- kaspresso/build.gradle.kts | 3 ++- .../device/screenshots/ScreenshotsImpl.kt | 10 +++++++--- .../files/resources/ResourcesDirsProvider.kt | 2 +- .../impl/DefaultResourcesDirsProvider.kt | 9 +++++++-- .../visual/DefaultVisualTestWatcher.kt | 7 ++++++- .../kaspresso/kaspresso/Kaspresso.kt | 1 + .../kaspresso/visual/VisualTestParams.kt | 19 +++++++++++++++++++ .../customdirectory/FlatDirectoryProvider.kt | 9 +++++++-- 8 files changed, 50 insertions(+), 10 deletions(-) diff --git a/kaspresso/build.gradle.kts b/kaspresso/build.gradle.kts index 679ec613a..a2215e00a 100644 --- a/kaspresso/build.gradle.kts +++ b/kaspresso/build.gradle.kts @@ -9,7 +9,8 @@ android { namespace = "com.kaspersky.kaspresso" defaultConfig { - buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE") ?: property("kaspresso.visualTestType").toString()) // [Record, Compare] + buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE") + ?: property("kaspresso.visualTestType")?.toString() ?: "Record") // [Record, Compare] } } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt index d647b945b..8510a92ff 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt @@ -6,6 +6,8 @@ import com.kaspersky.kaspresso.files.dirs.DirsProvider import com.kaspersky.kaspresso.files.extensions.FileExtension import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider import com.kaspersky.kaspresso.files.resources.ResourceFilesProvider +import com.kaspersky.kaspresso.files.resources.ResourcesDirNameProvider +import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider import com.kaspersky.kaspresso.logger.UiTestLogger import com.kaspersky.kaspresso.visual.ScreenshotsComparator import com.kaspersky.kaspresso.visual.VisualTestParams @@ -23,6 +25,7 @@ class ScreenshotsImpl( private val visualTestParams: VisualTestParams, private val dirsProvider: DirsProvider, private val resourceFileNamesProvider: ResourceFileNamesProvider, + private val resourcesDirsProvider: ResourcesDirsProvider, ) : Screenshots { private val originalScreenshotsDir: File @@ -59,7 +62,7 @@ class ScreenshotsImpl( null } else { originalScreenshotsDir.mkdirs() - File(originalScreenshotsDir, resourceFileNamesProvider.getFileName(tag, FileExtension.PNG.toString())) + resourcesDirsProvider.provide(originalScreenshotsDir).resolve(resourceFileNamesProvider.getFileName(tag, FileExtension.PNG.toString())) } doTakeAndApply(tag = tag, isFull = isFullWindow, targetPath) { screenshot = this } @@ -72,10 +75,11 @@ class ScreenshotsImpl( private fun File.compare() { logger.i("Comparing screenshot ${this.absolutePath}") - val originalScreenshot = File(originalScreenshotsDir, name) + val originalScreenshot = resourcesDirsProvider.provide(originalScreenshotsDir, provideCleared = false) + .resolve(resourceFileNamesProvider.getFileName(nameWithoutExtension, FileExtension.PNG.toString())) assert(originalScreenshot.exists()) { "Tried to assert screenshot $absolutePath, but failed to find matching " + - "original screenshot by the path: ${originalScreenshotsDir.absolutePath}/$name" + "original screenshot by the path: ${originalScreenshot.absolutePath}/$name" } val doesMatch = screenshotsComparator.compare(originalScreenshot, this) logger.i("Does screenshot $name matches the original one: $doesMatch") diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesDirsProvider.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesDirsProvider.kt index 14bf08f34..68318907c 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesDirsProvider.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/ResourcesDirsProvider.kt @@ -6,5 +6,5 @@ import java.io.File * Provides directories for resources */ interface ResourcesDirsProvider { - fun provide(dest: File, subDir: String? = null): File + fun provide(dest: File, subDir: String? = null, provideCleared: Boolean = true): File } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesDirsProvider.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesDirsProvider.kt index 72006706f..0f2cd1737 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesDirsProvider.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/files/resources/impl/DefaultResourcesDirsProvider.kt @@ -5,6 +5,7 @@ import com.kaspersky.kaspresso.files.extensions.findTestMethod import com.kaspersky.kaspresso.files.models.TestMethod import com.kaspersky.kaspresso.files.resources.ResourcesDirNameProvider import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider +import com.kaspersky.kaspresso.internal.extensions.other.createDirIfNeeded import java.io.File class DefaultResourcesDirsProvider( @@ -13,10 +14,14 @@ class DefaultResourcesDirsProvider( private val testThread: Thread = Thread.currentThread() ) : ResourcesDirsProvider { - override fun provide(dest: File, subDir: String?): File { + override fun provide(dest: File, subDir: String?, provideCleared: Boolean): File { val rootDir: File = dirsProvider.provideNew(dest) val resourcesDest: File = resolveResourcesDirDest(rootDir, subDir) - return dirsProvider.provideCleared(resourcesDest) + return if (provideCleared) { + dirsProvider.provideCleared(resourcesDest) + } else { + resourcesDest.createDirIfNeeded() + } } private fun resolveResourcesDirDest(rootDir: File, subDir: String? = null): File { diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt index f71c1752a..3cbd6dfdb 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultVisualTestWatcher.kt @@ -3,6 +3,7 @@ package com.kaspersky.kaspresso.internal.visual import com.kaspersky.kaspresso.device.files.Files import com.kaspersky.kaspresso.files.dirs.DirsProvider import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider +import com.kaspersky.kaspresso.internal.exceptions.AdbServerException import com.kaspersky.kaspresso.logger.Logger import com.kaspersky.kaspresso.visual.VisualTestParams import com.kaspersky.kaspresso.visual.VisualTestType @@ -32,7 +33,11 @@ internal class DefaultVisualTestWatcher( dirsProvider.provideCleared(diffDir) dirsProvider.provideCleared(originalScreenshotsTargetDir) - files.push(params.hostScreenshotsDir, dirsProvider.provideNew(File("")).absolutePath) + try { + files.push(params.hostScreenshotsDir, dirsProvider.provideNew(File("")).absolutePath) + } catch (ex: AdbServerException) { + throw RuntimeException("Failed to push screenshots. Please, check that they exist by the path: ${params.hostScreenshotsDir} (relatively to the ADB server executable", ex) + } logger.i("Done pushing the screenshots unto the device") } } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt index f19d8e85e..a37612bc1 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt @@ -797,6 +797,7 @@ data class Kaspresso( screenshotsComparator = screenshotsComparator, dirsProvider = dirsProvider, resourceFileNamesProvider = resourceFileNamesProvider, + resourcesDirsProvider = resourcesDirsProvider, ) } diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt index 9aa67dfe9..087c1325d 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt @@ -2,10 +2,29 @@ package com.kaspersky.kaspresso.visual import android.annotation.SuppressLint +/** + * @see com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator + */ @SuppressLint("SdCardPath") data class VisualTestParams( + /** + * Controls whether to take the new reference screenshots and save them or use the old ones and compare them to the ones being taken during the test + */ val testType: VisualTestType = VisualTestType.Record, + /** + * The path with the reference screenshots. Used to save the new reference screenshots if testType is set to the VisualTestType.Record + * or to push screenshot files if testType is set to VisualTestType.Compare + */ val hostScreenshotsDir: String = "original_screenshots", + /** + * The color threshold to mark the single pixel different from the other one. + * @see com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator.checkColors + */ val colorTolerance: Int = 1, + /** + * Controls the threshold of the screenshots difference. The value is in percents. Screenshots with difference less than this value + * are "acceptable" and don't fail the test + * @see com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator.compare + */ val tolerance: Float = 0.3f, ) diff --git a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/docloc_tests/customdirectory/FlatDirectoryProvider.kt b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/docloc_tests/customdirectory/FlatDirectoryProvider.kt index 6d05e9578..85c31cd79 100644 --- a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/docloc_tests/customdirectory/FlatDirectoryProvider.kt +++ b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/docloc_tests/customdirectory/FlatDirectoryProvider.kt @@ -4,6 +4,7 @@ import com.kaspersky.kaspresso.files.dirs.DirsProvider import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirsProvider import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider import com.kaspersky.kaspresso.files.resources.impl.DefaultResourcesDirNameProvider +import com.kaspersky.kaspresso.internal.extensions.other.createDirIfNeeded import java.io.File internal class FlatDirectoryProvider( @@ -12,12 +13,16 @@ internal class FlatDirectoryProvider( dirsProvider = dirsProvider, resourcesDirNameProvider = DefaultResourcesDirNameProvider() ) { - override fun provide(dest: File, subDir: String?): File { + override fun provide(dest: File, subDir: String?, provideCleared: Boolean): File { val rootDir: File = dirsProvider.provideNew(dest) val resultsDir: File = when (subDir) { null -> rootDir.resolve("") else -> rootDir.resolve(subDir).resolve("") } - return dirsProvider.provideCleared(resultsDir) + return if (provideCleared) { + dirsProvider.provideCleared(resultsDir) + } else { + resultsDir.createDirIfNeeded() + } } } From 35eb8c6061824d9e8802937322747cb84a5350b7 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Wed, 20 Nov 2024 16:05:25 +0300 Subject: [PATCH 06/10] TECH: Fix CustomizedSimpleTest compilation --- .../kaspressample/simple_tests/CustomizedSimpleTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt index 238c471c6..d657a25af 100644 --- a/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt +++ b/samples/kaspresso-sample/src/androidTest/kotlin/com/kaspersky/kaspressample/simple_tests/CustomizedSimpleTest.kt @@ -54,7 +54,7 @@ class CustomizedSimpleTest : TestCase( dirsProvider = dirsProvider, resourcesDirNameProvider = resourcesDirNameProvider ) { - override fun provide(dest: File, subDir: String?): File = + override fun provide(dest: File, subDir: String?, provideCleared: Boolean): File = dirsProvider.provideCleared(dirsProvider.provideNew(dest)) } From 09830acb48e0ea91a644b28197f27a688ae51678 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Wed, 20 Nov 2024 16:05:36 +0300 Subject: [PATCH 07/10] TECH: Remove unused annotation --- .../kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt | 3 --- 1 file changed, 3 deletions(-) diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt index 087c1325d..f2bc93e3d 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/visual/VisualTestParams.kt @@ -1,11 +1,8 @@ package com.kaspersky.kaspresso.visual -import android.annotation.SuppressLint - /** * @see com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator */ -@SuppressLint("SdCardPath") data class VisualTestParams( /** * Controls whether to take the new reference screenshots and save them or use the old ones and compare them to the ones being taken during the test From 28ef5e6fbe82b709b74092ec8f2f26017dd724c2 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Wed, 20 Nov 2024 17:09:07 +0300 Subject: [PATCH 08/10] TECH: Use test name as diff path --- .../AllureSupportKaspressoBuilder.kt | 8 ++++++- .../visual/AllureScreenshotsComparator.kt | 8 ++++--- .../device/screenshots/ScreenshotsImpl.kt | 1 - .../visual/DefaultScreenshotsComparator.kt | 23 +++++++++++-------- .../kaspresso/kaspresso/Kaspresso.kt | 8 ++++++- 5 files changed, 32 insertions(+), 16 deletions(-) diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt index 55c2e4b3e..1fb546909 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/AllureSupportKaspressoBuilder.kt @@ -115,7 +115,13 @@ private fun Kaspresso.Builder.initVisualTestParams(visualParams: VisualTestParam testLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_TEST_LOGGER_TAG) libLogger = UiTestLoggerImpl(Kaspresso.DEFAULT_LIB_LOGGER_TAG) - screenshotsComparator = AllureScreenshotsComparator(visualTestParams, testLogger, dirsProvider, resourcesRootDirsProvider) + screenshotsComparator = AllureScreenshotsComparator( + visualTestParams, + testLogger, + resourcesRootDirsProvider, + resourcesDirsProvider, + resourceFileNamesProvider, + ) adbServer = AdbServerImpl(LogLevel.WARN, libLogger) files = FilesImpl(libLogger, adbServer) visualTestWatcher = AllureVisualTestWatcher(visualTestParams, testLogger, (dirsProvider as AllureDirsProvider), resourcesRootDirsProvider, files) diff --git a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt index 5f66ac524..49cbc1ce8 100644 --- a/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt +++ b/allure-support/src/main/kotlin/com/kaspersky/components/alluresupport/visual/AllureScreenshotsComparator.kt @@ -2,7 +2,8 @@ package com.kaspersky.components.alluresupport.visual import android.graphics.Bitmap import com.kaspersky.components.alluresupport.files.attachScreenshotToAllureReport -import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider +import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider import com.kaspersky.kaspresso.internal.visual.DefaultScreenshotsComparator import com.kaspersky.kaspresso.logger.Logger @@ -12,9 +13,10 @@ import java.io.File class AllureScreenshotsComparator( visualTestParams: VisualTestParams, logger: Logger, - dirsProvider: DirsProvider, resourcesRootDirsProvider: ResourcesRootDirsProvider, -) : DefaultScreenshotsComparator(visualTestParams, logger, dirsProvider, resourcesRootDirsProvider) { + resourcesDirsProvider: ResourcesDirsProvider, + resourceFileNamesProvider: ResourceFileNamesProvider, +) : DefaultScreenshotsComparator(visualTestParams, logger, resourcesRootDirsProvider, resourcesDirsProvider, resourceFileNamesProvider) { override fun compare(originalScreenshot: File, newScreenshot: File): Boolean { val doScreenshotsMatch = super.compare(originalScreenshot, newScreenshot) if (!doScreenshotsMatch) { diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt index 8510a92ff..a33cba435 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/device/screenshots/ScreenshotsImpl.kt @@ -6,7 +6,6 @@ import com.kaspersky.kaspresso.files.dirs.DirsProvider import com.kaspersky.kaspresso.files.extensions.FileExtension import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider import com.kaspersky.kaspresso.files.resources.ResourceFilesProvider -import com.kaspersky.kaspresso.files.resources.ResourcesDirNameProvider import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider import com.kaspersky.kaspresso.logger.UiTestLogger import com.kaspersky.kaspresso.visual.ScreenshotsComparator diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt index e7fcb5642..375cc7c1f 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/internal/visual/DefaultScreenshotsComparator.kt @@ -3,7 +3,9 @@ package com.kaspersky.kaspresso.internal.visual import android.graphics.Bitmap import android.graphics.BitmapFactory import android.graphics.Color -import com.kaspersky.kaspresso.files.dirs.DirsProvider +import com.kaspersky.kaspresso.files.extensions.FileExtension +import com.kaspersky.kaspresso.files.resources.ResourceFileNamesProvider +import com.kaspersky.kaspresso.files.resources.ResourcesDirsProvider import com.kaspersky.kaspresso.files.resources.ResourcesRootDirsProvider import com.kaspersky.kaspresso.logger.Logger import com.kaspersky.kaspresso.visual.ScreenshotsComparator @@ -15,8 +17,9 @@ import kotlin.math.abs open class DefaultScreenshotsComparator( private val visualTestParams: VisualTestParams, private val logger: Logger, - private val dirsProvider: DirsProvider, private val resourcesRootDirsProvider: ResourcesRootDirsProvider, + private val resourcesDirsProvider: ResourcesDirsProvider, + private val resourceFileNamesProvider: ResourceFileNamesProvider, ) : ScreenshotsComparator { @Suppress("MagicNumber") @@ -79,20 +82,20 @@ open class DefaultScreenshotsComparator( val height = original.height val diffBitmap = Bitmap.createBitmap(width, height, original.config) diffBitmap.setPixels(diffPixels, 0, width, 0, 0, width, height) - val diffDir = dirsProvider.provideNew(resourcesRootDirsProvider.screenshotsDiffRootDir) - val screenshotDiff = File(diffDir, diffName) val scaledBitmap = Bitmap.createScaledBitmap( diffBitmap, width, height, false, ) - assert( - scaledBitmap.compress( - Bitmap.CompressFormat.PNG, - QUALITY, - FileOutputStream(screenshotDiff), - ) + + val screenshotDiff = resourcesDirsProvider.provide(resourcesRootDirsProvider.screenshotsDiffRootDir) + .resolve(resourceFileNamesProvider.getFileName(diffName, FileExtension.PNG.toString())) + + scaledBitmap.compress( + Bitmap.CompressFormat.PNG, + QUALITY, + FileOutputStream(screenshotDiff), ) return screenshotDiff diff --git a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt index a37612bc1..8a60af144 100644 --- a/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt +++ b/kaspresso/src/main/kotlin/com/kaspersky/kaspresso/kaspresso/Kaspresso.kt @@ -779,7 +779,13 @@ data class Kaspresso( if (!::clickParams.isInitialized) clickParams = ClickParams.default() if (!::artifactsPullParams.isInitialized) artifactsPullParams = ArtifactsPullParams(enabled = false) if (!::visualTestParams.isInitialized) visualTestParams = VisualTestParams(testType = VisualTestType.valueOf(BuildConfig.VISUAL_TEST_TYPE)) - if (!::screenshotsComparator.isInitialized) screenshotsComparator = DefaultScreenshotsComparator(visualTestParams, testLogger, dirsProvider, resourcesRootDirsProvider) + if (!::screenshotsComparator.isInitialized) screenshotsComparator = DefaultScreenshotsComparator( + visualTestParams, + testLogger, + resourcesRootDirsProvider, + resourcesDirsProvider, + resourceFileNamesProvider + ) if (!::visualTestWatcher.isInitialized) visualTestWatcher = DefaultVisualTestWatcher(visualTestParams, libLogger, dirsProvider, resourcesRootDirsProvider, files) if (!::screenshots.isInitialized) { From 71c1346b6817b686f32f18b1378471a292dab3b1 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Wed, 20 Nov 2024 17:09:54 +0300 Subject: [PATCH 09/10] TECH: Don't crash sync if gradle property not set --- kaspresso/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kaspresso/build.gradle.kts b/kaspresso/build.gradle.kts index a2215e00a..48d664d60 100644 --- a/kaspresso/build.gradle.kts +++ b/kaspresso/build.gradle.kts @@ -10,7 +10,7 @@ android { defaultConfig { buildConfigField("String", "VISUAL_TEST_TYPE", System.getenv("VISUAL_TEST_TYPE") - ?: property("kaspresso.visualTestType")?.toString() ?: "Record") // [Record, Compare] + ?: findProperty("kaspresso.visualTestType")?.toString() ?: "\"Record\"") // [Record, Compare] } } From ddb34ad46cd4abc56bfcea711f524f0dc3447d41 Mon Sep 17 00:00:00 2001 From: Nikita Evdokimov Date: Fri, 6 Dec 2024 16:34:49 +0300 Subject: [PATCH 10/10] TECH: PR comment --- kaspresso/gradle.properties | 1 - samples/kaspresso-sample/gradle.properties | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 samples/kaspresso-sample/gradle.properties diff --git a/kaspresso/gradle.properties b/kaspresso/gradle.properties index 3b77fb9dd..de6cebcd4 100644 --- a/kaspresso/gradle.properties +++ b/kaspresso/gradle.properties @@ -2,4 +2,3 @@ publish.artifactGroup=com.kaspersky.android-components publish.artifactName=kaspresso publish.publicationName=Kaspresso publish.bintrayRepo=Kaspresso -kaspresso.visualTestType="Compare" diff --git a/samples/kaspresso-sample/gradle.properties b/samples/kaspresso-sample/gradle.properties new file mode 100644 index 000000000..10df4d973 --- /dev/null +++ b/samples/kaspresso-sample/gradle.properties @@ -0,0 +1 @@ +kaspresso.visualTestType="Compare"