Test report reworked (#522)

Removed tsv. Sarif and detailed statistics appears only in event log. Bubble with warnings is now of warning type.
UnitTestBot · Jul 18, 2022 · 0a4ac22 · 0a4ac22
1 parent 9b95f55
commit 0a4ac22
Show file tree

Hide file tree

Showing 6 changed files with 150 additions and 161 deletions.
diff --git a/utbot-cli/src/main/kotlin/org/utbot/cli/GenerateTestsCommand.kt b/utbot-cli/src/main/kotlin/org/utbot/cli/GenerateTestsCommand.kt
@@ -145,7 +145,7 @@ class GenerateTestsCommand :
                     SourceFindingStrategyDefault(classFqn, sourceCodeFile, testsFilePath, projectRootPath)
                 val report = SarifReport(testSets, testClassBody, sourceFinding).createReport()
                 saveToFile(report, sarifReport)
-                println("The report was saved to \"$sarifReport\". You can open it using the VS Code extension \"Sarif Viewer\".")
+                println("The report was saved to \"$sarifReport\".")
             }
         }
     } catch (t: Throwable) {

diff --git a/.../main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt b/.../main/kotlin/org/utbot/framework/codegen/model/constructor/tree/CgTestClassConstructor.kt
@@ -189,9 +189,29 @@ data class TestsGenerationReport(
         get() = executables.firstOrNull()?.clazz
             ?: error("No executables found in test report")
 
-    // Summary message is generated lazily to avoid evaluation of classUnderTest
-    var summaryMessage: () -> String = { "Unit tests for $classUnderTest were generated successfully." }
     val initialWarnings: MutableList<() -> String> = mutableListOf()
+    val hasWarnings: Boolean
+        get() = initialWarnings.isNotEmpty()
+
+    val detailedStatistics: String
+        get() = buildString {
+            appendHtmlLine("Class: ${classUnderTest.qualifiedName}")
+            val testMethodsStatistic = executables.map { it.countTestMethods() }
+            val errors = executables.map { it.countErrors() }
+            val overallErrors = errors.sum()
+
+            appendHtmlLine("Successful test methods: ${testMethodsStatistic.sumBy { it.successful }}")
+            appendHtmlLine(
+                "Failing because of unexpected exception test methods: ${testMethodsStatistic.sumBy { it.failing }}"
+            )
+            appendHtmlLine(
+                "Failing because of exceeding timeout test methods: ${testMethodsStatistic.sumBy { it.timeout }}"
+            )
+            appendHtmlLine(
+                "Failing because of possible JVM crash test methods: ${testMethodsStatistic.sumBy { it.crashes }}"
+            )
+            appendHtmlLine("Not generated because of internal errors test methods: $overallErrors")
+        }
 
     fun addMethodErrors(testSet: UtMethodTestSet, errors: Map<String, Int>) {
         this.errors[testSet.method] = errors
@@ -216,61 +236,24 @@ data class TestsGenerationReport(
         }
     }
 
-    override fun toString(): String = buildString {
-        appendHtmlLine(summaryMessage())
-        appendHtmlLine()
-        initialWarnings.forEach { appendHtmlLine(it()) }
-        appendHtmlLine()
+    fun toString(isShort: Boolean): String = buildString {
+        appendHtmlLine("Target: ${classUnderTest.qualifiedName}")
+        if (initialWarnings.isNotEmpty()) {
+            initialWarnings.forEach { appendHtmlLine(it()) }
+            appendHtmlLine()
+        }
 
         val testMethodsStatistic = executables.map { it.countTestMethods() }
-        val errors = executables.map { it.countErrors() }
         val overallTestMethods = testMethodsStatistic.sumBy { it.count }
-        val overallErrors = errors.sum()
+
         appendHtmlLine("Overall test methods: $overallTestMethods")
-        appendHtmlLine("Successful test methods: ${testMethodsStatistic.sumBy { it.successful }}")
-        appendHtmlLine(
-            "Failing because of unexpected exception test methods: ${testMethodsStatistic.sumBy { it.failing }}"
-        )
-        appendHtmlLine(
-            "Failing because of exceeding timeout test methods: ${testMethodsStatistic.sumBy { it.timeout }}"
-        )
-        appendHtmlLine(
-            "Failing because of possible JVM crash test methods: ${testMethodsStatistic.sumBy { it.crashes }}"
-        )
-        appendHtmlLine("Not generated because of internal errors test methods: $overallErrors")
-    }
 
-    // TODO: should we use TsvWriter from univocity instead of this manual implementation?
-    fun getFileContent(): String =
-        (listOf(getHeader()) + getLines()).joinToString(System.lineSeparator())
-
-    private fun getHeader(): String {
-        val columnNames = listOf(
-            "Executable/Number of test methods",
-            SUCCESSFUL,
-            FAILING,
-            TIMEOUT,
-            CRASH,
-            "Errors tests"
-        )
-
-        return columnNames.joinToString(TAB_SEPARATOR)
+        if (!isShort) {
+            appendHtmlLine(detailedStatistics)
+        }
     }
 
-    private fun getLines(): List<String> =
-        executables.map { executable ->
-            val testMethodStatistic = executable.countTestMethods()
-            with(testMethodStatistic) {
-                listOf(
-                    executable,
-                    successful,
-                    failing,
-                    timeout,
-                    crashes,
-                    executable.countErrors()
-                ).joinToString(TAB_SEPARATOR)
-            }
-        }
+    override fun toString(): String = toString(false)
 
     private fun UtMethod<*>.countTestMethods(): TestMethodStatistic = TestMethodStatistic(
         testMethodsNumber(successfulExecutions),
@@ -291,9 +274,4 @@ data class TestsGenerationReport(
     private data class TestMethodStatistic(val successful: Int, val failing: Int, val timeout: Int, val crashes: Int) {
         val count: Int = successful + failing + timeout + crashes
     }
-
-    companion object {
-        private const val TAB_SEPARATOR: String = "\t"
-        const val EXTENSION: String = ".tsv"
-    }
 }
diff --git a/...ork/src/main/kotlin/org/utbot/framework/plugin/sarif/GenerateTestsAndSarifReportFacade.kt b/...ork/src/main/kotlin/org/utbot/framework/plugin/sarif/GenerateTestsAndSarifReportFacade.kt
@@ -53,7 +53,6 @@ class GenerateTestsAndSarifReportFacade(
             mergedSarifReportFile.writeText(mergedReport)
             if (verbose) {
                 println("SARIF report was saved to \"${mergedSarifReportFile.path}\"")
-                println("You can open it using the VS Code extension \"Sarif Viewer\"")
             }
         }
     }

diff --git a/...-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerationController.kt b/...-intellij/src/main/kotlin/org/utbot/intellij/plugin/generator/CodeGenerationController.kt
@@ -15,7 +15,6 @@ import com.intellij.openapi.editor.Editor
 import com.intellij.openapi.project.DumbService
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.util.Computable
-import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.psi.JavaDirectoryService
 import com.intellij.psi.PsiClass
 import com.intellij.psi.PsiClassOwner
@@ -47,7 +46,6 @@ import org.jetbrains.kotlin.psi.psiUtil.endOffset
 import org.jetbrains.kotlin.psi.psiUtil.startOffset
 import org.jetbrains.kotlin.scripting.resolve.classId
 import org.utbot.common.HTML_LINE_SEPARATOR
-import org.utbot.common.PathUtil.classFqnToPath
 import org.utbot.common.PathUtil.toHtmlLinkTag
 import org.utbot.common.appendHtmlLine
 import org.utbot.framework.codegen.Import
@@ -56,7 +54,6 @@ import org.utbot.framework.codegen.RegularImport
 import org.utbot.framework.codegen.StaticImport
 import org.utbot.framework.codegen.model.CodeGenerator
 import org.utbot.framework.codegen.model.TestsCodeWithTestReport
-import org.utbot.framework.codegen.model.constructor.tree.TestsGenerationReport
 import org.utbot.framework.plugin.api.CodegenLanguage
 import org.utbot.framework.plugin.api.UtMethod
 import org.utbot.framework.plugin.api.UtMethodTestSet
@@ -70,16 +67,17 @@ import org.utbot.intellij.plugin.models.GenerateTestsModel
 import org.utbot.intellij.plugin.models.packageName
 import org.utbot.intellij.plugin.sarif.SarifReportIdea
 import org.utbot.intellij.plugin.sarif.SourceFindingStrategyIdea
+import org.utbot.intellij.plugin.ui.DetailsTestsReportNotifier
 import org.utbot.intellij.plugin.ui.SarifReportNotifier
 import org.utbot.intellij.plugin.ui.TestReportUrlOpeningListener
 import org.utbot.intellij.plugin.ui.TestsReportNotifier
+import org.utbot.intellij.plugin.ui.WarningTestsReportNotifier
 import org.utbot.intellij.plugin.ui.utils.getOrCreateSarifReportsPath
 import org.utbot.intellij.plugin.ui.utils.getOrCreateTestResourcesPath
 import org.utbot.intellij.plugin.ui.utils.showErrorDialogLater
 import org.utbot.intellij.plugin.util.signature
 import org.utbot.sarif.SarifReport
 import java.nio.file.Path
-import java.nio.file.Paths
 import java.util.concurrent.CountDownLatch
 import java.util.concurrent.TimeUnit
 import kotlin.reflect.KClass
@@ -167,7 +165,6 @@ object CodeGenerationController {
         SarifReportNotifier.notify(
             info = """
                 SARIF report was saved to ${toHtmlLinkTag(mergedReportFile.path)}$HTML_LINE_SEPARATOR
-                You can open it using the VS Code extension "Sarif Viewer"
             """.trimIndent()
         )
     }
@@ -394,32 +391,10 @@ object CodeGenerationController {
             "Test resources directory $testResourcesDirPath does not exist"
         }
 
-        val testReportSubDir = "utbot-tests-report"
-        val classFqn = with(testsCodeWithTestReport.testsGenerationReport.classUnderTest) {
-            qualifiedName ?: error("Could not save tests report for anonymous or local class $this")
-        }
-        val fileReportPath = classFqnToPath(classFqn)
-
-        val resultedReportedPath =
-            Paths.get(
-                testResourcesDirPath.toString(),
-                testReportSubDir,
-                fileReportPath + "TestReport" + TestsGenerationReport.EXTENSION
-            )
-
-        val parent = resultedReportedPath.parent
-        requireNotNull(parent) {
-            "Expected from parent of $resultedReportedPath to be not null but it is null"
-        }
-
-        VfsUtil.createDirectories(parent.toString())
-        resultedReportedPath.toFile().writeText(testsCodeWithTestReport.testsGenerationReport.getFileContent())
-
         processInitialWarnings(testsCodeWithTestReport, model)
 
         val notifyMessage = buildString {
-            appendHtmlLine(testsCodeWithTestReport.testsGenerationReport.toString())
-            appendHtmlLine()
+            appendHtmlLine(testsCodeWithTestReport.testsGenerationReport.toString(isShort = true))
             val classUnderTestPackageName =
                 testsCodeWithTestReport.testsGenerationReport.classUnderTest.classId.packageFqName.toString()
             if (classUnderTestPackageName != model.testPackageName) {
@@ -430,12 +405,19 @@ object CodeGenerationController {
                 appendHtmlLine(warningMessage)
                 appendHtmlLine()
             }
-            val savedFileMessage = """
-                Tests report was saved to ${toHtmlLinkTag(resultedReportedPath.toString())} in TSV format
+            val eventLogMessage = """
+                <a href="${TestReportUrlOpeningListener.prefix}${TestReportUrlOpeningListener.eventLogSuffix}">See details in Event Log</a>.
             """.trimIndent()
-            appendHtmlLine(savedFileMessage)
+            appendHtmlLine(eventLogMessage)
+        }
+
+        if (testsCodeWithTestReport.testsGenerationReport.hasWarnings) {
+            WarningTestsReportNotifier.notify(notifyMessage)
+        } else {
+            TestsReportNotifier.notify(notifyMessage)
         }
-        TestsReportNotifier.notify(notifyMessage)
+
+        DetailsTestsReportNotifier.notify(testsCodeWithTestReport.testsGenerationReport.detailedStatistics)
     }
 
     private fun processInitialWarnings(testsCodeWithTestReport: TestsCodeWithTestReport, model: GenerateTestsModel) {
@@ -445,8 +427,6 @@ object CodeGenerationController {
         }
 
         testsCodeWithTestReport.testsGenerationReport.apply {
-            summaryMessage = { "Unit tests for $classUnderTest were generated with warnings.<br>" }
-
             if (model.forceMockHappened) {
                 initialWarnings.add {
                     """