Improve shutdown handler logic in reftest mode

Remove `[cylc][[reference test]]suite shutdown event handler`.
Fix more flaky tests.

cylc/flow/cfgspec/suite.py

@@ -91,7 +91,6 @@
             'mail footer': [VDR.V_STRING],
         },
         'reference test': {
-            'suite shutdown event handler': [VDR.V_STRING, '(reftest)'],
             'required run mode': [
                 VDR.V_STRING,
                 '', 'live', 'simulation', 'dummy-local', 'dummy'],
@@ -289,6 +288,9 @@ def upg(cfg, descr):
     u.obsolete('7.8.1', ['runtime', '__MANY__', 'events', 'reset timer'])
     u.obsolete('8.0.0', ['cylc', 'log resolved dependencies'])
     u.obsolete('8.0.0', ['cylc', 'reference test', 'allow task failures'])
+    u.obsolete(
+        '8.0.0',
+        ['cylc', 'reference test', 'suite shutdown event handler'])
     u.deprecate(
         '8.0.0',
         ['cylc', 'abort if any task fails'],

cylc/flow/config.py

@@ -2271,9 +2271,9 @@ def get_expected_failed_tasks(self):
           in reference test mode.
         - None if there is no expectation either way.
         """
-        if self.cfg['cylc']['events']['abort if any task fails']:
-            return []
-        elif self.options.reftest:
+        if self.options.reftest:
             return self.cfg['cylc']['reference test']['expected task failures']
+        elif self.cfg['cylc']['events']['abort if any task fails']:
+            return []
         else:
             return None

cylc/flow/suite_events.py

@@ -127,8 +127,8 @@ def _run_event_custom_handlers(self, config, ctx):
         # 1. Handlers for specific event
         # 2. General handlers
         handlers = self.get_events_conf(config, '%s handler' % ctx.event)
-        if ctx.event == self.EVENT_SHUTDOWN and handlers == ['(reftest)']:
-            return self._run_reftest(config, ctx)
+        if config.options.reftest and ctx.event == self.EVENT_SHUTDOWN:
+            self._run_reftest(config, ctx)
         if not handlers and (
                 ctx.event in
                 self.get_events_conf(config, 'handler events', [])):
@@ -203,15 +203,6 @@ def configure_reftest(cls, config):
         req = rtc['required run mode']
         if req and not config.run_mode(req):
             raise SchedulerError('suite allows only %s reference tests' % req)
-        handlers = cls.get_events_conf(config, 'shutdown handler')
-        if handlers:
-            LOG.warning('shutdown handlers replaced by reference test')
-        config.cfg['cylc']['events']['shutdown handler'] = (
-            [rtc['suite shutdown event handler']])
-        config.cfg['cylc']['events']['abort if shutdown handler fails'] =\
-            True
-        if not rtc['expected task failures']:
-            config.cfg['cylc']['events']['abort if any task fails'] = True
         config.cfg['cylc']['events']['abort on timeout'] = True
         timeout = rtc['%s mode suite timeout' % config.run_mode()]
         config.cfg['cylc']['events'][cls.EVENT_TIMEOUT] = timeout

flakytests/cylc-take-checkpoints/00-basic.t

@@ -30,7 +30,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
 
 # Suite reloads+inserts new task to mess up prerequisites - suite should stall
 suite_run_ok "${TEST_NAME_BASE}-run" \
-    timeout 120 cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
+    cylc run --debug --no-detach --reference-test "${SUITE_NAME}"
 cylc ls-checkpoints "${SUITE_NAME}" | date-remove >'cylc-ls-checkpoints.out'
 contains_ok 'cylc-ls-checkpoints.out' <<'__OUT__'
 #######################################################################

flakytests/events/01-task.t

@@ -18,13 +18,16 @@
 # Validate and run the task events suite.
 . "$(dirname "$0")/test_header"
 #-------------------------------------------------------------------------------
-set_test_number 2
+set_test_number 3
 #-------------------------------------------------------------------------------
 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
 #-------------------------------------------------------------------------------
 run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
 suite_run_ok "${TEST_NAME_BASE}-run" \
     cylc run --reference-test --debug --no-detach "${SUITE_NAME}"
+sed 's/ (after .*)$//' "${SUITE_RUN_DIR}/log/suite/events.log" | sort -u \
+    >'events.log.sorted'
+cmp_ok 'events.log.sorted' 'events.log'
 #-------------------------------------------------------------------------------
 purge_suite "${SUITE_NAME}"
 exit

flakytests/events/01-task/events.log

@@ -1,12 +1,12 @@
 EVENT                TASK     MESSAGE
+critical             foo.1    failed/EXIT
+execution timeout    foo.1    execution timeout after PT3S
+failed               baz.1    job failed
 retry                foo.1    job failed, retrying in PT3S
+started              baz.1    job started
+submission failed    bar.1    job submission failed
 submission retry     bar.1    job submission failed, submit-retrying in PT3S
-submitted            baz.1    job submitted
 submission timeout   baz.1    submission timeout after PT3S
-submission failed    bar.1    job submission failed
-execution timeout    foo.1    execution timeout after PT3S
-started              baz.1    job started
-failed               baz.1    job failed
-warning              foo.1    this is a user-defined warning message
-critical             foo.1    failed/EXIT
+submitted            baz.1    job submitted
 succeeded            foo.1    job succeeded
+warning              foo.1    this is a user-defined warning message

flakytests/events/01-task/reference.log

@@ -6,4 +6,4 @@
 2013/05/20 22:05:00 INFO - [baz.1] -triggered off ['prep.1']
 2013/05/20 22:05:03 INFO - [foo.1] -triggered off ['prep.1']
 2013/05/20 22:05:03 INFO - [bar.1] -triggered off ['prep.1']
-2013/05/20 22:05:07 INFO - [done.1] -triggered off ['bar.1', 'baz.1']
+2013/05/20 22:05:07 INFO - [done.1] -triggered off ['bar.1', 'baz.1', 'foo.1']

flakytests/events/01-task/suite.rc

@@ -9,18 +9,20 @@
 [cylc]
     [[environment]]
         EVNTLOG = {{ EVNTLOG }}
+    [[events]]
+        abort on stalled = True
+        abort on inactivity = True
+        inactivity = PT3M
     [[reference test]]
-        live mode suite timeout = PT1M
-        suite shutdown event handler = log-check.sh
         expected task failures = bar.1, baz.1
 
 [scheduling]
     [[graph]]
         R1 = """
-          prep => foo & bar & baz
-      bar:submit-fail & baz:fail & foo => done
-           done => !bar & !baz
-                """
+            prep => foo & bar & baz
+            bar:submit-fail & baz:fail & foo => done
+            done => !bar & !baz
+        """
 [runtime]
     [[root]]
         # make the event handler log available to gcylc:

tests/authentication/05-full-control.t

@@ -17,10 +17,10 @@
 
 # Test authentication - privilege 'full-control' (with passphrase).
 
-. $(dirname $0)/test_header
+. "$(dirname "$0")/test_header"
 set_test_number 12
 
-install_suite "${TEST_NAME_BASE}" basic
+install_suite "${TEST_NAME_BASE}" 'basic'
 
 TEST_NAME="${TEST_NAME_BASE}-validate"
 run_ok "${TEST_NAME}" cylc validate "${SUITE_NAME}"
@@ -168,20 +168,20 @@ __END__
 TEST_NAME="${TEST_NAME_BASE}-show1"
 run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}"
 cylc log "${SUITE_NAME}" > suite.log1
-grep_ok "\[client-command\] get_suite_info ${USER}@.*:cylc-show" suite.log1
+grep_ok "\\[client-command\\] get_suite_info ${USER}@.*cylc-show" 'suite.log1'
 
 # "cylc show" (task info) OK.
 TEST_NAME="${TEST_NAME_BASE}-show2"
 run_ok "${TEST_NAME}" cylc show "${SUITE_NAME}" foo
 cylc log "${SUITE_NAME}" > suite.log2
-grep_ok "\[client-command\] get_task_info ${USER}@.*:cylc-show" suite.log2
+grep_ok "\\[client-command\\] get_task_info ${USER}@.*cylc-show" 'suite.log2'
 
 # Commands OK.
 # (Reset to same state).
 TEST_NAME="${TEST_NAME_BASE}-trigger"
 run_ok "${TEST_NAME}" cylc reset "${SUITE_NAME}" -s failed foo 1
 cylc log "${SUITE_NAME}" > suite.log3
-grep_ok "\[client-command\] reset_task_states ${USER}@.*:cylc-reset" suite.log3
+grep_ok "\\[client-command\\] reset_task_states ${USER}@.*cylc-reset" 'suite.log3'
 
 # Shutdown and purge.
 TEST_NAME="${TEST_NAME_BASE}-stop"

tests/cylc-scan/03-monitor.t

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #-------------------------------------------------------------------------------
 # Test cylc monitor USER_AT_HOST interface, using cylc scan output.
-. $(dirname $0)/test_header
+. "$(dirname "$0")/test_header"
 #-------------------------------------------------------------------------------
 set_test_number 6
 #-------------------------------------------------------------------------------
@@ -33,16 +33,21 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${SUITE_NAME}"
 run_ok "${TEST_NAME_BASE}-run" cylc run "${SUITE_NAME}"
 
 TEST_NAME="${TEST_NAME_BASE}-monitor-1"
+# Need fields from "cylc scan", cannot quote
+# shellcheck disable=SC2046
 run_ok "${TEST_NAME}" cylc monitor \
     $(cylc scan --color=never -n "${SUITE_NAME}") --once
 grep_ok "${SUITE_NAME} - 1 task" "${TEST_NAME}.stdout"
 
 # Same again, but force a port scan instead of looking under ~/cylc-run.
 # (This also tests GitHub #2795 -"cylc scan -a" abort).
 TEST_NAME="${TEST_NAME_BASE}-monitor-2"
+# Need fields from "cylc scan", cannot quote
+# shellcheck disable=SC2046
 run_ok "${TEST_NAME}" cylc monitor \
     $(cylc scan --color=never -n "${SUITE_NAME}") --once
 grep_ok "${SUITE_NAME} - 1 task" "${TEST_NAME}.stdout"
 
-cylc stop --kill "${SUITE_NAME}"
+cylc stop --kill --max-polls=20 --interval=1 "${SUITE_NAME}"
 purge_suite "${SUITE_NAME}"
+exit

tests/events/18-suite-event-mail/suite.rc

@@ -9,8 +9,9 @@
         mail footer = see: http://localhost/stuff/%(owner)s/%(suite)s/
         mail smtp = {{MAIL_SMTP}}
 {% endif %}{# not GLOBALCFG is not defined #}
-    [[reference test]]
-        live mode suite timeout=PT1M
+        abort on stalled = True
+        abort on inactivity = True
+        inactivity = PT3M
 
 [scheduling]
     [[graph]]