added the option to use a pyproject.toml to control cylc lint settings.

CHANGES.md

@@ -28,34 +28,21 @@ creating a new release entry be sure to copy & paste the span tag with the
 `actions:bind` attribute, which is used by a regex to find the text to be
 updated. Only the first match gets replaced, so it's fine to leave the old
 ones in. -->
--------------------------------------------------------------------------------
 
+-------------------------------------------------------------------------------
 ## __cylc-8.1.0 (<span actions:bind='release-date'>Upcoming</span>)__
 
 ### Enhancements
 
 [#5032](https://github.com/cylc/cylc-flow/pull/5032) - set a default limit of
 100 for the "default" queue.
 
-[#5055](https://github.com/cylc/cylc-flow/pull/5055) - Hard-code the serial
-numbers of Cylc Lint's style issues and allow users to ignore Cylc Lint issues
-using `--ignore <Issue Code>`.
-
--------------------------------------------------------------------------------
-## __cylc-8.0.2 (<span actions:bind='release-date'>Upcoming</span>)__
-
-Maintenance release.
-
-### Fixes
-
-[#5067](https://github.com/cylc/cylc-flow/pull/5067) - Datastore fix for
-taskdefs removed before restart.
+[#5055](https://github.com/cylc/cylc-flow/pull/5055) and
+[#5086](https://github.com/cylc/cylc-flow/pull/5086) - Upgrades to `cylc lint`
+- Allow users to ignore Cylc Lint issues using `--ignore <Issue Code>`.
+- Allow settings for `cylc lint` to be recorded in a pyproject.toml file.
+- Allow files to be excluded from `cylc lint` checks.
 
-[#5066](https://github.com/cylc/cylc-flow/pull/5066) - Fix bug where
-.cylcignore only found if `cylc install` is run in source directory.
-
-[#5091](https://github.com/cylc/cylc-flow/pull/5091) - Fix problems with
-tutorial workflows.
 
 -------------------------------------------------------------------------------
 ## __cylc-8.0.1 (<span actions:bind='release-date'>Released 2022-08-16</span>)__
@@ -64,9 +51,6 @@ Maintenance release.
 
 ### Fixes
 
-[#5025](https://github.com/cylc/cylc-flow/pull/5025) - Fix a bug where polling
-causes a failed task to be shown as submitted when the workflow is reloaded.
-
 [#5045](https://github.com/cylc/cylc-flow/pull/5045) -
 Fix issue where unsatisfied xtriggers could be wiped on reload.
 
@@ -1347,4 +1331,3 @@ None.
 -------------------------------------------------------------------------------
 
 **For changes prior to Cylc 8, see https://github.com/cylc/cylc-flow/blob/7.8.x/CHANGES.md**
-

INSTALL.md

@@ -5,7 +5,7 @@
 **See the [Cylc installation guide](https://cylc.github.io/cylc-doc/stable/html/installation.html)
 for more detailed information.**
 
-Cylc must be installed on workflow and job hosts. Third-party dependencies
+Cylc must be installed on workflow and task job hosts. Third-party dependencies
 are not required on job hosts.
 
 ## Cylc 7

conda-environment.yml

@@ -18,6 +18,8 @@ dependencies:
   - pyzmq >=22,<23
   - setuptools >=49
   - urwid >=2,<3
+  # Add # [py<3.11] for tomli once Python 3.11 Released
+  - tomli >=2.0.0
 
 # optional dependencies
   #- empy >=3.3,<3.4

cylc/flow/cfgspec/globalcfg.py

@@ -355,7 +355,7 @@
 - slurm_packjob
 - moab
 
-Directives are written to the top of the job script in the correct format
+Directives are written to the top of the task job script in the correct format
 for the job runner.
 
 Specifying directives individually like this allows use of default directives
@@ -1082,7 +1082,7 @@ def default_for(
         A platform consists of a group of one or more hosts which share a
         file system and a job runner (batch system).
 
-        A platform must allow interaction with the same job from *any*
+        A platform must allow interaction with the same task job from *any*
         of its hosts.
 
         .. versionadded:: 8.0.0
@@ -1535,7 +1535,7 @@ def default_for(
                 We recommend using a clean job submission environment for
                 consistent handling of local and remote jobs. However,
                 this is not the default behavior because it prevents
-                local jobs from running, unless ``$PATH`` contains the
+                local task jobs from running, unless ``$PATH`` contains the
                 ``cylc`` wrapper script.
 
                 Specific environment variables can be singled out to pass
@@ -1655,7 +1655,7 @@ def default_for(
         all be suitable for a given job.
 
 
-        When Cylc submits a job it will pick a platform from a group.
+        When Cylc sets up a task job it will pick a platform from a group.
         Cylc will then use the selected platform for all interactions with
         that job.
 

cylc/flow/cfgspec/workflow.py

@@ -970,7 +970,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 If no parents are listed default is ``root``.
             ''')
             Conf('script', VDR.V_STRING, desc=dedent('''
-                The main custom script run from the job script.
+                The main custom script run from the task job script.
 
                 It can be an external command or script, or inlined scripting.
 
@@ -980,7 +980,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 this='script', example='my_script.sh'
             ))
             Conf('init-script', VDR.V_STRING, desc=dedent('''
-                Custom script run by the job script before the task
+                Custom script run by the task job script before the task
                 execution environment is configured.
 
                 By running before the task execution environment is configured,
@@ -992,7 +992,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 this should no longer be necessary.
             ''') + get_script_common_text(this='init-script'))
             Conf('env-script', VDR.V_STRING, desc=dedent('''
-                Custom script run by the job script between the
+                Custom script run by the task job script between the
                 cylc-defined environment (workflow and task identity, etc.) and
                 the user-defined task runtime environment.
 
@@ -1001,10 +1001,10 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 It can be an external command or script, or inlined scripting.
             ''') + get_script_common_text(this='env-script'))
             Conf('err-script', VDR.V_STRING, desc=('''
-                Script run when a job error is detected.
+                Script run when a task job error is detected.
 
                 Custom script to be run at the end of the error trap,
-                which is triggered due to failure of a command in the job
+                which is triggered due to failure of a command in the task job
                 script or trappable job kill.
 
                 The output of this script will always
@@ -1029,7 +1029,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 this='exit-script', example='rm -f "$TMP_FILES"'
             ))
             Conf('pre-script', VDR.V_STRING, desc=dedent('''
-                Custom script run by the job script immediately
+                Custom script run by the task job script immediately
                 before :cylc:conf:`[..]script`.
 
                 The pre-script can be an external command or script, or
@@ -1039,7 +1039,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 example='echo "Hello from workflow ${CYLC_WORKFLOW_ID}!"'
             ))
             Conf('post-script', VDR.V_STRING, desc=dedent('''
-                Custom script run by the job script immediately
+                Custom script run by the task job script immediately
                 after :cylc:conf:`[..]script`.
 
                 The post-script can be an external
@@ -1049,7 +1049,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
             Conf('work sub-directory', VDR.V_STRING, desc='''
                 The directory from which tasks are executed.
 
-                Job scripts are executed from within *work directories*
+                Task job scripts are executed from within *work directories*
                 created automatically under the workflow run directory. A task
                 can get its own work directory from ``$CYLC_TASK_WORK_DIR``
                 (or ``$PWD`` if it does not ``cd`` elsewhere at
@@ -1083,13 +1083,13 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 )
             )
             Conf('execution retry delays', VDR.V_INTERVAL_LIST, None, desc=f'''
-                Cylc can automate resubmission of a failed job.
+                Cylc can automate resubmission of a failed task job.
 
                 Execution retry delays are a list of ISO 8601
                 durations/intervals which tell Cylc how long to wait before
                 resubmitting a failed job.
 
-                Each time Cylc resubmits a job it will increment the
+                Each time Cylc resubmits a task job it will increment the
                 variable ``$CYLC_TASK_TRY_NUMBER`` in the task execution
                 environment. ``$CYLC_TASK_TRY_NUMBER`` allows you to vary task
                 behavior between submission attempts.
@@ -1101,13 +1101,13 @@ def get_script_common_text(this: str, example: Optional[str] = None):
             ''')
             Conf('execution time limit', VDR.V_INTERVAL, desc=f'''
                 Set the execution (:term:`wallclock <wallclock time>`) time
-                limit of a job.
+                limit of a task job.
 
                 For ``background`` and ``at`` job runners Cylc runs the
                 job's script using the timeout command. For other job runners
                 Cylc will convert execution time limit to a :term:`directive`.
 
-                If a job exceeds its execution time limit Cylc can
+                If a task job exceeds its execution time limit Cylc can
                 poll the job multiple times. You can set polling
                 intervals using :cylc:conf:`global.cylc[platforms]
                 [<platform name>]execution time limit polling intervals`
@@ -1573,7 +1573,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                 The user defined task execution environment.
 
                 Variables defined here can refer to cylc workflow and task
-                identity variables, which are exported earlier in the job
+                identity variables, which are exported earlier in the task job
                 script. Variable assignment expressions can use cylc
                 utility commands because access to cylc is also configured
                 earlier in the script.
@@ -1591,7 +1591,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
 
                     The order of definition is preserved that each variable can
                     refer to previously defined
-                    variables. Values are passed through to the job
+                    variables. Values are passed through to the task job
                     script without evaluation or manipulation by Cylc
                     (with the exception of valid Python string templates
                     that match parameterized task names - see below), so any

cylc/flow/data_store_mgr.py

@@ -818,15 +818,12 @@ def generate_ghost_task(self, tp_id, itask, is_parent=False):
         if is_orphan:
             self.generate_orphan_task(itask)
 
-        # Most of the time the definition node will be in the store.
+        # Most the time the definition node will be in the store,
+        # so use try/except.
         try:
             task_def = self.data[self.workflow_id][TASKS][t_id]
         except KeyError:
-            try:
-                task_def = self.added[TASKS][t_id]
-            except KeyError:
-                # Task removed from workflow definition.
-                return False
+            task_def = self.added[TASKS][t_id]
 
         update_time = time()
         tp_stamp = f'{tp_id}@{update_time}'

cylc/flow/etc/job.sh

@@ -19,11 +19,11 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 ###############################################################################
-# Bash functions for a cylc job.
+# Bash functions for a cylc task job.
 ###############################################################################
 
 ###############################################################################
-# The main function for a cylc job.
+# The main function for a cylc task job.
 cylc__job__main() {
     # Export CYLC_ workflow and task environment variables
     cylc__job__inst__cylc_env
@@ -50,7 +50,7 @@ cylc__job__main() {
         trap "cylc__job_vacation ${signal_name}" "${signal_name}"
     done
     set -euo pipefail
-    # Write job self-identify
+    # Write task job self-identify
     USER="${USER:-$(whoami)}"
     typeset host="${HOSTNAME:-}"
     if [[ -z "${host}" ]]; then
@@ -65,7 +65,7 @@ cylc__job__main() {
     # We were using a HERE document for writing info here until we notice that
     # Bash uses temporary files for HERE documents, which can be inefficient.
     echo "Workflow : ${CYLC_WORKFLOW_ID}"
-    echo "Job : ${CYLC_TASK_JOB} (try ${CYLC_TASK_TRY_NUMBER})"
+    echo "Task Job : ${CYLC_TASK_JOB} (try ${CYLC_TASK_TRY_NUMBER})"
     echo "User@Host: ${USER}@${host}"
     echo
     # Derived environment variables
@@ -229,7 +229,7 @@ cylc__job__poll_grep_workflow_log() {
 }
 
 ###############################################################################
-# Run a function in the job instance file, if possible.
+# Run a function in the task job instance file, if possible.
 # Arguments:
 #   func_name: name of function without the "cylc__job__inst__" prefix
 # Returns:

cylc/flow/etc/tutorial/cylc-forecasting-workflow/bin/get-observations

@@ -39,6 +39,7 @@ import os
 import re
 
 import requests
+import urllib3
 
 
 BASE_URL = ('http://datapoint.metoffice.gov.uk/public/data/'
@@ -75,20 +76,31 @@ def get_datapoint_data(site_id, time, api_key):
     """Get weather data from the DataPoint service."""
     # The URL required to get the data.
     time = datetime.strptime(time, '%Y%m%dT%H%MZ').strftime('%Y-%m-%dT%H:%MZ')
-    url = BASE_URL.format(time=time, site_id=site_id, api_key=api_key)
-    req = requests.get(url)
-    if req.status_code != 200:
-        raise Exception(f'{url} returned exit code {req.status_code}')
+    url = BASE_URL.format(time=time, site_id=site_id,
+                          api_key=api_key)
+
+    # HTTP header to prevent 403 error:
+    hdr = {'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 '
+                          '(KHTML, like Gecko) Chrome/23.0.1271.64 '
+                          'Safari/537.11'),
+           'Accept': ('text/html,application/xhtml+xml,application/xml;'
+                      'q=0.9,*/*;q=0.8'),
+           'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
+           'Accept-Encoding': 'none',
+           'Accept-Language': 'en-US,en;q=0.8',
+           'Connection': 'keep-alive'}
+    http = urllib3.PoolManager()
+    req = http.request('GET', url, headers=hdr)
+    if req.status != 200:
+        raise Exception(f'{url} returned exit code {req.status}')
     # Get the data and parse it as JSON.
     print('Opening URL: %s' % url)
-    return req.json()['SiteRep']['DV']['Location']
+    return json.loads(req.data)['SiteRep']['DV']['Location']
 
 
 def get_archived_data(site_id, time):
     """Return archived weather data from the workflow directory."""
-    print(
-        [os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json']
-    )
+    print([os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json'])
     path = os.path.join(
         os.environ['CYLC_WORKFLOW_RUN_DIR'], 'data', time, site_id + '.json')
     print('Opening File: %s' % path)
@@ -105,7 +117,7 @@ def extract_observations(data):
 
 def reference_lat_long(site_id):
     """Extract lat-long from a reference file."""
-    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
     info = Path(src).read_text()
     siteinfo = re.findall(f'^{int(site_id):05d}.*', info, re.MULTILINE)
     if not siteinfo:
@@ -165,7 +177,7 @@ def synop_grab(site_id, time):
 def get_nearby_site(site_id, badsites):
     """Use Pythagoras to find the next nearest site."""
     lat, lon = reference_lat_long(site_id)
-    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR'], 'etc/met-office-sites.dat')
+    src = Path(os.environ['CYLC_WORKFLOW_RUN_DIR']) / 'etc/met-office-sites.dat'
     info = Path(src).read_text()
     dist = 100
     result = 0
@@ -183,6 +195,7 @@ def get_nearby_site(site_id, badsites):
     return int(result[0]), dist
 
 
+
 def main(site_id, api_key=None):
     cycle_point = os.environ['CYLC_TASK_CYCLE_POINT']
 
@@ -209,6 +222,7 @@ def main(site_id, api_key=None):
 
         obs = extract_observations(data)
 
+
     # Write observations.
     with open('wind.csv', 'w+') as data_file:
         data_file.write(', '.join(obs))