Use the rhts-report-result alias for reporting beakerlib subresul…

…ts (#3372) * Use rhts-report-result alias instead of tmt-report-result for reporting beakerlib subresults * Add and improve subresult tests to cover also the rhts-report-result calls * Do not depend on an order of TMT_REPORT_RESULT_SCRIPT aliases * Change log file permissions in `tmt-report-result` Co-authored-by: Petr Šplíchal <[email protected]> Co-authored-by: Miloš Prchlík <[email protected]>
teemtee · Dec 10, 2024 · 789b604 · 789b604
1 parent 94951ab
commit 789b604
Show file tree

Hide file tree

Showing 5 changed files with 83 additions and 34 deletions.
diff --git a/tests/execute/result/subresults.sh b/tests/execute/result/subresults.sh
@@ -38,29 +38,54 @@ rlJournalStart
         rlAssertGrep "warn /extra-tmt-report-result/weird" "$rlRun_LOG"
         rlAssertGrep "skip /extra-tmt-report-result/skip" "$rlRun_LOG"
 
+        # Extra call of rhts-report-result in the beakerlib test phase
+        rlAssertGrep "fail /extra-rhts-report-result/bad" "$rlRun_LOG"
+
         # The phase itself must also exists as a subresult and it should pass,
-        # even one of its extra tmt-report-result reported a FAIL. The phase
-        # outcome evaluation is based on beakerlib test framework.
+        # even one or more of its extra {tmt,rhts}-report-result reported a
+        # FAIL. The phase outcome evaluation is based on beakerlib test
+        # framework.
         rlAssertGrep "pass /phase-test-multiple-tmt-report-result" "$rlRun_LOG"
 
-
         # Check the shell framework subtests outcomes
-        rlAssertGrep "pass /fail-subtest/good" "$rlRun_LOG"
-        rlAssertGrep "fail /fail-subtest/fail" "$rlRun_LOG"
-        rlAssertGrep "warn /fail-subtest/weird" "$rlRun_LOG"
-        rlAssertGrep "skip /fail-subtest/skip" "$rlRun_LOG"
-
         rlAssertGrep "pass /pass-subtest/good0" "$rlRun_LOG"
         rlAssertGrep "pass /pass-subtest/good1" "$rlRun_LOG"
         rlAssertGrep "pass /pass-subtest/good2" "$rlRun_LOG"
+        rlAssertGrep "pass /pass-subtest/good3" "$rlRun_LOG"
 
         rlAssertGrep "pass /skip-subtest/extra-pass" "$rlRun_LOG"
-        rlAssertGrep "skip /skip-subtest/extra-skip" "$rlRun_LOG"
+        rlAssertGrep "skip /skip-subtest/extra-skip1" "$rlRun_LOG"
+        rlAssertGrep "skip /skip-subtest/extra-skip2" "$rlRun_LOG"
+
+        rlAssertGrep "pass /fail-subtest/good" "$rlRun_LOG"
+        rlAssertGrep "fail /fail-subtest/fail" "$rlRun_LOG"
+        rlAssertGrep "warn /fail-subtest/weird" "$rlRun_LOG"
+        rlAssertGrep "skip /fail-subtest/skip" "$rlRun_LOG"
+        rlAssertGrep "fail /fail-subtest/fail-rhts" "$rlRun_LOG"
 
 
         # Check the subresults get correctly saved in results.yaml
         rlRun "results_file=${run_dir}/plan/execute/results.yaml"
 
+        rlRun "yq -ey '.[] | select(.name == \"/test/shell/pass\") | .subresult' ${results_file} > subresults_pass.yaml"
+        rlAssertGrep "name: /pass-subtest/good0" "subresults_pass.yaml"
+        rlAssertGrep "name: /pass-subtest/good1" "subresults_pass.yaml"
+        rlAssertGrep "name: /pass-subtest/good2" "subresults_pass.yaml"
+        rlAssertGrep "name: /pass-subtest/good3" "subresults_pass.yaml"
+        rlAssertNotGrep "original-result: \(fail\|skip\|warn\)" "subresults_pass.yaml"
+
+        rlRun "yq -ey '.[] | select(.name == \"/test/shell/skip\") | .subresult' ${results_file} > subresults_skip.yaml"
+        rlAssertGrep "name: /skip-subtest/extra-pass" "subresults_skip.yaml"
+        rlAssertGrep "name: /skip-subtest/extra-skip1" "subresults_skip.yaml"
+        rlAssertGrep "name: /skip-subtest/extra-skip2" "subresults_skip.yaml"
+
+        rlRun "yq -ey '.[] | select(.name == \"/test/shell/fail\") | .subresult' ${results_file} > subresults_fail.yaml"
+        rlAssertGrep "name: /fail-subtest/good" "subresults_fail.yaml"
+        rlAssertGrep "name: /fail-subtest/fail" "subresults_fail.yaml"
+        rlAssertGrep "name: /fail-subtest/weird" "subresults_fail.yaml"
+        rlAssertGrep "name: /fail-subtest/skip" "subresults_fail.yaml"
+        rlAssertGrep "name: /fail-subtest/fail-rhts" "subresults_fail.yaml"
+
         rlRun "yq -ey '.[] | select(.name == \"/test/beakerlib\") | .subresult' ${results_file} > subresults_beakerlib.yaml"
         rlAssertGrep "name: /phase-setup" "subresults_beakerlib.yaml"
         rlAssertGrep "name: /phase-test-pass" "subresults_beakerlib.yaml"
@@ -70,54 +95,53 @@ rlJournalStart
         rlAssertGrep "name: /extra-tmt-report-result/bad" "subresults_beakerlib.yaml"
         rlAssertGrep "name: /extra-tmt-report-result/weird" "subresults_beakerlib.yaml"
         rlAssertGrep "name: /extra-tmt-report-result/skip" "subresults_beakerlib.yaml"
+        rlAssertGrep "name: /extra-rhts-report-result/bad" "subresults_beakerlib.yaml"
 
-        rlRun "yq -ey '.[] | select(.name == \"/test/shell/fail\") | .subresult' ${results_file} > subresults_fail.yaml"
-        rlAssertGrep "name: /fail-subtest/good" "subresults_fail.yaml"
-        rlAssertGrep "name: /fail-subtest/fail" "subresults_fail.yaml"
-        rlAssertGrep "name: /fail-subtest/weird" "subresults_fail.yaml"
-        rlAssertGrep "name: /fail-subtest/skip" "subresults_fail.yaml"
-
-        rlRun "yq -ey '.[] | select(.name == \"/test/shell/pass\") | .subresult' ${results_file} > subresults_pass.yaml"
-        rlAssertGrep "name: /pass-subtest/good0" "subresults_pass.yaml"
-        rlAssertGrep "name: /pass-subtest/good1" "subresults_pass.yaml"
-        rlAssertGrep "name: /pass-subtest/good2" "subresults_pass.yaml"
-        rlAssertNotGrep "original-result: \(fail\|skip\|warn\)" "subresults_pass.yaml"
 
         # Check the subresults log entries are set in results.yaml
         rlAssertGrep "- data/.*/extra-tmt-report-result_good_bkr_good_log$" "subresults_beakerlib.yaml"
         rlAssertGrep "- data/.*/extra-tmt-report-result_bad_bkr_bad_log$" "subresults_beakerlib.yaml"
         rlAssertGrep "- data/.*/extra-tmt-report-result_weird_bkr_weird_log$" "subresults_beakerlib.yaml"
         rlAssertGrep "- data/.*/extra-tmt-report-result_skip_bkr_skip_log$" "subresults_beakerlib.yaml"
+        rlAssertGrep "- data/.*/extra-rhts-report-result_bad_bkr_rhts_bad_log$" "subresults_beakerlib.yaml"
 
         rlAssertGrep "- data/.*/fail-subtest_good_good_log$" "subresults_fail.yaml"
         rlAssertGrep "- data/.*/fail-subtest_fail_fail_log$" "subresults_fail.yaml"
         rlAssertGrep "- data/.*/fail-subtest_weird_weird_log$" "subresults_fail.yaml"
         rlAssertGrep "- data/.*/fail-subtest_skip_skip_log$" "subresults_fail.yaml"
+        rlAssertGrep "- data/.*/fail-subtest_fail-rhts_fail-rhts_log$" "subresults_fail.yaml"
+
+        rlAssertGrep "- data/.*/skip-subtest_extra-skip2_skip-rhts_log$" "subresults_skip.yaml"
 
         rlAssertGrep "- data/.*/pass-subtest_good0_good0_log$" "subresults_pass.yaml"
         rlAssertGrep "- data/.*/pass-subtest_good1_good1_log$" "subresults_pass.yaml"
         rlAssertGrep "- data/.*/pass-subtest_good2_good2_log$" "subresults_pass.yaml"
+        rlAssertGrep "- data/.*/pass-subtest_good3_good3_log$" "subresults_pass.yaml"
+
 
         # Check the subresults log files actually exist
         rlRun "log_dir=$run_dir/plan/execute/data/guest/default-0/test"
         rlAssertExists "$log_dir/shell/pass-3/data/pass-subtest_good0_good0_log"
         rlAssertExists "$log_dir/shell/pass-3/data/pass-subtest_good1_good1_log"
         rlAssertExists "$log_dir/shell/pass-3/data/pass-subtest_good2_good2_log"
+        rlAssertExists "$log_dir/shell/pass-3/data/pass-subtest_good3_good3_log"
 
         rlAssertExists "$log_dir/shell/fail-2/data/fail-subtest_fail_fail_log"
         rlAssertExists "$log_dir/shell/fail-2/data/fail-subtest_good_good_log"
         rlAssertExists "$log_dir/shell/fail-2/data/fail-subtest_skip_skip_log"
         rlAssertExists "$log_dir/shell/fail-2/data/fail-subtest_weird_weird_log"
+        rlAssertExists "$log_dir/shell/fail-2/data/fail-subtest_fail-rhts_fail-rhts_log"
 
         rlAssertExists "$log_dir/beakerlib-1/data/extra-tmt-report-result_bad_bkr_bad_log"
         rlAssertExists "$log_dir/beakerlib-1/data/extra-tmt-report-result_good_bkr_good_log"
         rlAssertExists "$log_dir/beakerlib-1/data/extra-tmt-report-result_skip_bkr_skip_log"
         rlAssertExists "$log_dir/beakerlib-1/data/extra-tmt-report-result_weird_bkr_weird_log"
+        rlAssertExists "$log_dir/beakerlib-1/data/extra-rhts-report-result_bad_bkr_rhts_bad_log"
         rlAssertExists "$log_dir/beakerlib-1/data/journal.xml"
     rlPhaseEnd
 
     rlPhaseStartCleanup
-        rlRun "rm subresults_{beakerlib,fail,pass}.yaml"
+        rlRun "rm subresults_{beakerlib,fail,pass,skip}.yaml"
         rlRun "popd"
         rlRun "rm -rf $run_dir" 0 "Remove run directory"
     rlPhaseEnd

diff --git a/tests/execute/result/subresults/beaker-phases-subresults.sh b/tests/execute/result/subresults/beaker-phases-subresults.sh
@@ -12,7 +12,7 @@ rlJournalStart
         rlRun "test -n \"\$TMT_TEST_SERIAL_NUMBER\" -o -n \"\$TESTID\"" 0 "Check the variables are not empty"
         rlAssertEquals "TESTID must be set to TMT_TEST_SERIAL_NUMBER" "$TESTID" "$TMT_TEST_SERIAL_NUMBER"
 
-        rlRun "[[ \$BEAKERLIB_COMMAND_REPORT_RESULT =~ tmt-report-result$ ]]" 0 "Check the variable contains path to a tmt-report-result script"
+        rlRun "[[ \$BEAKERLIB_COMMAND_REPORT_RESULT =~ rhts-report-result$ ]]" 0 "Check the variable contains path to a rhts-report-result script"
     rlPhaseEnd
 
     rlPhaseStartTest "phase-test pass"
@@ -26,16 +26,16 @@ rlJournalStart
     rlPhaseEnd
 
     rlPhaseStartTest "phase-test multiple tmt-report-result"
-        rlRun "touch bkr_good_log"
-        rlRun "touch bkr_bad_log"
-        rlRun "touch bkr_weird_log"
-        rlRun "touch bkr_skip_log"
+        rlRun "touch bkr_{good,bad,weird,skip,rhts_bad}_log"
 
         # This will create more subresults for each tmt-report-result call
         rlRun "tmt-report-result -o bkr_good_log extra-tmt-report-result/good PASS"
         rlRun "tmt-report-result -o bkr_bad_log extra-tmt-report-result/bad FAIL"
         rlRun "tmt-report-result -o bkr_weird_log extra-tmt-report-result/weird WARN"
         rlRun "tmt-report-result -o bkr_skip_log extra-tmt-report-result/skip SKIP"
+
+        # We also support reporting the subresult via rhts-report-result alias
+        rlRun "rhts-report-result extra-rhts-report-result/bad FAIL bkr_rhts_bad_log"
     rlPhaseEnd
 
     rlPhaseStartCleanup "phase-cleanup"

diff --git a/tests/execute/result/subresults/test.fmf b/tests/execute/result/subresults/test.fmf
@@ -5,11 +5,15 @@
         touch /tmp/good1_log; tmt-report-result -o /tmp/good1_log /pass-subtest/good1 PASS
         touch /tmp/good2_log; tmt-report-result -o /tmp/good2_log /pass-subtest/good2 PASS
 
+        # The log file for rhts-report-result must be always provided
+        touch /tmp/good3_log; rhts-report-result /pass-subtest/good3 PASS /tmp/good3_log
+
 /shell/skip:
-    summary: Test with skipped and pass test should pass
+    summary: Test with skipped and pass tests should pass
     test: |
         tmt-report-result /skip-subtest/extra-pass PASS
-        tmt-report-result /skip-subtest/extra-skip SKIP
+        tmt-report-result /skip-subtest/extra-skip1 SKIP
+        touch /tmp/skip-rhts_log; rhts-report-result /skip-subtest/extra-skip2 SKIP /tmp/skip-rhts_log
 
 /shell/fail:
     summary: Reduced outcome of shell subresults must be fail
@@ -18,11 +22,9 @@
         touch /tmp/fail_log; tmt-report-result -o /tmp/fail_log /fail-subtest/fail FAIL
         touch /tmp/weird_log; tmt-report-result -o /tmp/weird_log /fail-subtest/weird WARN
         touch /tmp/skip_log; tmt-report-result -o /tmp/skip_log /fail-subtest/skip SKIP
+        touch /tmp/fail-rhts_log; rhts-report-result /fail-subtest/fail-rhts FAIL /tmp/fail-rhts_log
 
 /beakerlib:
     summary: Beakerlib rlPhaseEnd as a tmt subresult
-
-    # Explicitly set the test framework
     framework: beakerlib
-
     test: ./beaker-phases-subresults.sh
diff --git a/tmt/frameworks/beakerlib.py b/tmt/frameworks/beakerlib.py
@@ -8,13 +8,16 @@
 import tmt.utils
 from tmt.frameworks import TestFramework, provides_framework
 from tmt.result import ResultOutcome
-from tmt.utils import Environment, EnvVarValue, Path
+from tmt.utils import Environment, EnvVarValue, GeneralError, Path
 
 if TYPE_CHECKING:
     from tmt.base import DependencySimple, Test
     from tmt.steps.execute import TestInvocation
 
 
+BEAKERLIB_REPORT_RESULT_COMMAND = 'rhts-report-result'
+
+
 @provides_framework('beakerlib')
 class Beakerlib(TestFramework):
     @classmethod
@@ -33,6 +36,25 @@ def get_environment_variables(
             invocation: 'TestInvocation',
             logger: tmt.log.Logger) -> tmt.utils.Environment:
 
+        # The beakerlib calls the command in this variable in the following way:
+        # $BEAKERLIB_COMMAND_REPORT_RESULT "$testname" "$result" "$logfile" "$score"
+        # - https://github.com/beakerlib/beakerlib/blob/5a85937f557b735f32996eb55cd6b9a33f3fe653/src/testing.sh#L1076
+        #
+        # If we use the `tmt-report-result` value here, the script will not be compatible
+        # with the third `$logfile` positional argument - it will just ignore it because it
+        # accepts the logfile provided only by `-o/--outputFile` option be default.
+        #
+        # The reason the `rhts-report-result` alias is used here is a compatibility layer
+        # implemented by the script itself. If the script gets called with this name, it
+        # will *accept* the third positional argument as a `logfile`.
+        # - https://github.com/teemtee/tmt/blob/e7cf41d1fe5a4dcbb3270758586f41313e9462ec/tmt/steps/execute/scripts/tmt-report-result#L101
+        if BEAKERLIB_REPORT_RESULT_COMMAND not in [
+                tmt.steps.execute.TMT_REPORT_RESULT_SCRIPT.source_filename,
+                *tmt.steps.execute.TMT_REPORT_RESULT_SCRIPT.aliases]:
+            raise GeneralError("Beakerlib framework requires the "
+                               f"'{BEAKERLIB_REPORT_RESULT_COMMAND}' script to be available "
+                               "on a guest.")
+
         return Environment({
             'BEAKERLIB_DIR': EnvVarValue(invocation.path),
             'BEAKERLIB_COMMAND_SUBMIT_LOG': EnvVarValue(
@@ -41,8 +63,7 @@ def get_environment_variables(
 
             # The command in this variable gets called with every `rlPhaseEnd` call in beakerlib.
             'BEAKERLIB_COMMAND_REPORT_RESULT': EnvVarValue(
-                invocation.guest.scripts_path /
-                tmt.steps.execute.TMT_REPORT_RESULT_SCRIPT.source_filename),
+                invocation.guest.scripts_path / BEAKERLIB_REPORT_RESULT_COMMAND),
 
             # This variables must be set explicitly, otherwise the beakerlib `rlPhaseEnd` macro
             # will not call the the command in `BEAKERLIB_COMMAND_REPORT_RESULT`.

diff --git a/tmt/steps/execute/scripts/tmt-report-result b/tmt/steps/execute/scripts/tmt-report-result
@@ -29,6 +29,8 @@ copy_outputfile_to_data_dir () {
     [[ ${fileprefix:0:1} == '_' ]] && fileprefix=$(echo "$fileprefix" | cut -c 2-)
     # Copy outputfile to data dir. Prefix with underscored test name.
     cp -f "$outputFile" "${TMT_TEST_DATA}/${fileprefix}_${filename}"
+    # Make sure that the log file is readable for guest.pull()
+    chmod a+r "${TMT_TEST_DATA}/${fileprefix}_${filename}"
     outputFile="$fileprefix"_"$filename"
 }