STY: Fix build errors

.stylelintrc.yml

@@ -23,7 +23,6 @@ rules:
   no-duplicate-at-import-rules: true
   no-duplicate-selectors: true
   no-empty-source: true
-  no-extra-semicolons: true
   no-invalid-double-slash-comments: true
   property-no-unknown: true
   selector-pseudo-class-no-unknown: true

gramex/apps/admin2/gramexadmin.py

@@ -161,9 +161,9 @@ def evaluate(handler, code):
         context['handler'] = handler
         # B307:eval B102:exec_used is safe since only admin can run this
         if mode == 'eval':
-            result = eval(co, context)  # nosec B307
+            result = eval(co, context)  # noqa S307
         else:
-            exec(co, context)  # nosec B102
+            exec(co, context)  # noqa S102
             result = None
     except Exception as e:
         result = e
@@ -217,10 +217,10 @@ def system_information(handler):
     apps = {
         # B602:any_other_function_with_shell_equals_true is safe here since the code is
         # constructed entirely in this function. We use shell to pick up the commands' paths.
-        ('node', 'version'): Subprocess('node --version', shell=True),  # nosec 602
-        ('npm', 'version'): Subprocess('npm --version', shell=True),  # nosec 602
-        ('yarn', 'version'): Subprocess('yarn --version', shell=True),  # nosec 602
-        ('git', 'version'): Subprocess('git --version', shell=True),  # nosec 602
+        ('node', 'version'): Subprocess('node --version', shell=True),  # noqa S604
+        ('npm', 'version'): Subprocess('npm --version', shell=True),  # noqa S604
+        ('yarn', 'version'): Subprocess('yarn --version', shell=True),  # noqa S604
+        ('git', 'version'): Subprocess('git --version', shell=True),  # noqa S604
     }
     for key, proc in apps.items():
         stdout, stderr = yield proc.wait_for_exit()

gramex/apps/logviewer/logviewer.py

@@ -4,8 +4,8 @@
 from glob import glob
 
 # B410:import_lxml lxml.etree is safe on https://github.com/tiran/defusedxml/tree/main/xmltestdata
-from lxml.etree import Element  # nosec B410
-from lxml.html import fromstring, tostring  # nosec B410
+from lxml.etree import Element  # noqa S410
+from lxml.html import fromstring, tostring  # noqa S410
 import numpy as np
 import pandas as pd
 import gramex.data

gramex/apps/ui/__init__.py

@@ -8,7 +8,7 @@
 import gramex.handlers
 
 # B404:import_subprocess only for JS compilation
-import subprocess  # nosec B404
+import subprocess  # noqa S404
 from hashlib import md5
 from tornado.gen import coroutine, Return
 from functools import partial
@@ -63,7 +63,7 @@ def _get_cache_key(state):
     '''Return short string capturing state of object. Used to create unique filenames for state'''
     cache_key = gramex.cache.cache_key(state).encode('utf-8')
     # B303, B324:md5 is safe here - it's not for cryptographic use
-    return md5(cache_key).hexdigest()[:5]  # nosec B303, B324
+    return md5(cache_key).hexdigest()[:5]  # noqa S113
 
 
 @coroutine

gramex/cache.py

@@ -14,7 +14,7 @@
 import sqlalchemy as sa
 
 # B404:import_subprocess only developers can access this, not users
-import subprocess  # nosec B404
+import subprocess  # noqa B404
 import sys
 import tempfile
 import time
@@ -574,7 +574,7 @@ def urlfetch(url: str, info: bool = False, **kwargs: dict) -> Union[str, Dict]:
             return {'name': url, 'r': None, 'url': None, 'ext': ext, 'content_type': content_type}
         else:
             return url
-    r = requests.get(url, **kwargs)  # nosec B113 - timeout is controlled by kwargs
+    r = requests.get(url, **kwargs)  # noqa S113 - timeout is controlled by kwargs
     if 'Content-Type' in r.headers:
         content_type = r.headers['Content-Type'].split(';')[0]
         ext = mimetypes.guess_extension(content_type, strict=False)
@@ -688,7 +688,7 @@ def __init__(
         kwargs['close_fds'] = 'posix' in sys.builtin_module_names
 
         # B603:subprocess_without_shell_equals_true: only developers can access this, not users
-        self.proc = subprocess.Popen(args, **kwargs)  # nosec B603
+        self.proc = subprocess.Popen(args, **kwargs)  # noqa S603
         self.thread = {}  # Has the running threads
         self.future = {}  # Stores the futures indicating stream close
         self.loop = _get_current_ioloop()
@@ -872,15 +872,15 @@ def fetch_from_server(handler):
     # If process was never started, start it
     if key not in _daemons:
         # B404:import_subprocess only developers can access this, not users
-        started = _daemons[key] = Subprocess(args, **kwargs)  # nosec B404
+        started = _daemons[key] = Subprocess(args, **kwargs)  # noqa S404
 
     # Ensure that process is running. Restart if required
     proc = _daemons[key]
     restart = int(restart)
     while proc.proc.returncode is not None and restart > 0:
         restart -= 1
         # B404:import_subprocess only developers can access this, not users
-        proc = started = _daemons[key] = Subprocess(args, **kwargs)  # nosec B404
+        proc = started = _daemons[key] = Subprocess(args, **kwargs)  # noqa S404
     if proc.proc.returncode is not None:
         raise RuntimeError(f'Error {proc.proc.returncode} starting {arg_str}')
     if started:
@@ -1332,7 +1332,7 @@ def _yaml(handle, **kwargs):
 
     kwargs.setdefault('Loader', yaml.SafeLoader)
     # B506:yaml_load we load safely using SafeLoader
-    return yaml.load(handle.read(), **kwargs)  # nosec B506
+    return yaml.load(handle.read(), **kwargs)  # noqa S506
 
 
 def _template(path, **kwargs):
@@ -1449,20 +1449,20 @@ def _table_status(engine, tables):
             # Works only on MySQL 5.7 and above
             # B608:hardcoded_sql_expressions only used internally
             w = _wheres('table_schema', 'table_name', db, tables)
-            q = 'SELECT update_time FROM information_schema.tables WHERE ' + w  # nosec B608
+            q = 'SELECT update_time FROM information_schema.tables WHERE ' + w  # noqa S608
         elif dialect == 'snowflake':
             # https://docs.snowflake.com/en/sql-reference/info-schema/tables.html
             w = _wheres('table_schema', 'table_name', db, tables)
-            q = 'SELECT last_altered FROM information_schema.tables WHERE ' + w  # nosec B608
+            q = 'SELECT last_altered FROM information_schema.tables WHERE ' + w  # noqa S608
         elif dialect == 'mssql':
             # https://goo.gl/b4aL9m
             w = _wheres('database_id', 'object_id', db, tables, fn=['DB_ID', 'OBJECT_ID'])
-            q = 'SELECT last_user_update FROM sys.dm_db_index_usage_stats WHERE ' + w  # nosec B608
+            q = 'SELECT last_user_update FROM sys.dm_db_index_usage_stats WHERE ' + w  # noqa S608
         elif dialect == 'postgresql':
             # https://www.postgresql.org/docs/9.6/static/monitoring-stats.html
             w = _wheres('schemaname', 'relname', 'public', tables)
             q = (
-                'SELECT n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_all_tables '  # nosec B608
+                'SELECT n_tup_ins, n_tup_upd, n_tup_del FROM pg_stat_all_tables '  # noqa S608
                 + 'WHERE '
                 + w
             )

gramex/config.py

@@ -287,7 +287,7 @@ def used_kwargs(method, kwargs, ignore_keywords=False):
 def random_string(size, chars=_valid_key_chars):
     '''Return random string of length size using chars (which defaults to alphanumeric)'''
     # B311:random random() is safe since it's for non-cryptographic use
-    return ''.join(choice(chars) for index in range(size))  # nosec B311
+    return ''.join(choice(chars) for index in range(size))  # noqa S311
 
 
 class PathConfig(AttrDict):
@@ -561,7 +561,7 @@ def _yaml_open(path, default=AttrDict(), **kwargs):
     with path.open(encoding='utf-8') as handle:
         try:
             # B506:yaml_load we use a safe loader
-            result = yaml.load(handle, Loader=ConfigYAMLLoader)  # nosec B506
+            result = yaml.load(handle, Loader=ConfigYAMLLoader)  # noqa S506
         except Exception:
             app_log.exception(f'Config error: {path}')
             return default
@@ -610,7 +610,7 @@ def _yaml_open(path, default=AttrDict(), **kwargs):
             base, expr = key.split(' if ', 2)
             try:
                 # B307:eval this is safe since `expr` is written by app developer
-                condition = eval(expr, globals(), frozen_vars)  # nosec B307
+                condition = eval(expr, globals(), frozen_vars)  # noqa S307
             except Exception:
                 condition = False
                 app_log.exception(f'Failed condition evaluation: {key}')
@@ -897,7 +897,7 @@ def setup_secrets(path, max_age_days=1000000, clear=True):
         app_log.info(f'Fetching remote secrets from {secrets_url}')
         # Load string from the URL -- but ignore comments. file:// URLs are fine too
         # B310:urllib_urlopen secrets can be local files or URLs
-        value = yaml.safe_load(urlopen(secrets_url))  # nosec B310
+        value = yaml.safe_load(urlopen(secrets_url))  # noqa S310
         value = decode_signed_value(secrets_key, '', value, max_age_days=max_age_days)
         result.update(loads(value.decode('utf-8')))
     # If SECRETS_IMPORT: is set, fetch secrets from those file(s) as well.

gramex/data.py

@@ -1207,7 +1207,7 @@ def alter(
                 # Use eval() to handle direct types like INTEGER *and* expressions like VARCHAR(3)
                 # eval() is safe here since `col_type` is written by app developer
                 # B307:eval is safe here since `col_type` is written by app developer
-                row['type'] = eval(col_type.upper(), vars(sa.types))  # nosec B307
+                row['type'] = eval(col_type.upper(), vars(sa.types))  # noqa S307
             row['type_'] = row.pop('type')
             if 'default' in row:
                 from inspect import isclass
@@ -2076,7 +2076,7 @@ def _insert_mongodb(
 ):
     table = _mongodb_collection(url, database, collection, **kwargs)
     result = table.insert_many([_mongodb_json(row) for row in rows.to_dict(orient='records')])
-    meta['inserted'] = [{'id': str(id) for id in result.inserted_ids}]  # noqa: B035
+    meta['inserted'] = [{'id': str(id) for id in result.inserted_ids}]  # noqa B035
     return len(result.inserted_ids)
 
 

gramex/handlers/basehandler.py

@@ -787,7 +787,7 @@ def _write_headers(self, headers):
 
     def debug_exception(self, typ, value, tb):
         super(BaseHandler, self).log_exception(typ, value, tb)
-        import pdb  # noqa: T100
+        import pdb  # noqa T100
 
         pdb.post_mortem(tb)
 

gramex/handlers/capturehandler.py

@@ -10,7 +10,7 @@
 from threading import Thread, Lock
 
 # B404:import_subprocess only for JS compilation
-from subprocess import Popen, PIPE, STDOUT  # nosec B404
+from subprocess import Popen, PIPE, STDOUT  # noqa S404
 from urllib.parse import urlencode, urljoin
 from tornado.web import HTTPError
 from tornado.httpclient import AsyncHTTPClient
@@ -135,7 +135,7 @@ def _start(self):
             self.close()
             # B603:subprocess_without_shell_equals_true is safe since self.cmd is taken from
             # the YAML configuration (from developers)
-            self.proc = Popen(shlex.split(self.cmd), stdout=PIPE, stderr=STDOUT)  # nosec B603
+            self.proc = Popen(shlex.split(self.cmd), stdout=PIPE, stderr=STDOUT)  # noqa S603
             self.proc.poll()
             atexit.register(self.close)
             # TODO: what if readline() does not return quickly?

gramex/handlers/drivehandler.py

@@ -131,7 +131,7 @@ def post(self, *path_args, **path_kwargs):
             path = slug.filename(file)
             # B311:random random() is safe since it's for non-cryptographic use
             while self.fs.exists(path):
-                randomletter = choice(digits + ascii_lowercase)  # nosec B311
+                randomletter = choice(digits + ascii_lowercase)  # noqa S311
                 path = os.path.splitext(path)[0] + randomletter + ext
             self.args['file'][i] = file
             self.args['ext'][i] = ext.lower()

gramex/handlers/processhandler.py

@@ -114,7 +114,7 @@ def get(self, *path_args):
             self.cmdargs,
             # NOTE: developer should sanitize args if shell=True
             # B604 any_other_function_with_shell_equals_true
-            shell=self.shell,  # nosec B604
+            shell=self.shell,  # noqa S604
             cwd=self.cwd,
             stream_stdout=self.stream_stdout,
             stream_stderr=self.stream_stderr,

gramex/handlers/socialhandler.py

@@ -116,7 +116,7 @@ def get_token(self, key, fetch=lambda info, key, val: info.get(key, val)):
         token = self.kwargs.get(key, None)  # Get from config
         session_token = fetch(info, key, None)
         # B105:hardcoded_password_string: 'persist' is not a password
-        if token == 'persist':  # nosec B105
+        if token == 'persist':  # noqa S105
             token = self.read_store().get(key, None)  # If persist, use store
             if token is None and session_token:  # Or persist from session
                 self.write_store(info)

gramex/handlers/uploadhandler.py

@@ -148,7 +148,7 @@ def deletefiles(self, handler):
         for delete_key in self.keys.get('delete', []):
             for key in handler.args.get(delete_key, []):
                 stat = {'success': False, 'key': key}
-                if key in self.store.keys():  # noqa: SIM118 self.store is not iterable
+                if key in self.store.keys():  # noqa SIM118 self.store is not iterable
                     path = os.path.join(self.path, key)
                     if os.path.exists(path):
                         os.remove(path)

gramex/install.py

@@ -20,7 +20,7 @@
 from orderedattrdict.yamlutils import AttrDictYAMLLoader
 
 # B404:import_subprocess only developers can access this, not users
-from subprocess import Popen, check_output, CalledProcessError  # nosec B404
+from subprocess import Popen, check_output, CalledProcessError  # noqa S404
 from textwrap import dedent
 from tornado.template import Template
 from zipfile import ZipFile
@@ -531,7 +531,7 @@ def complexity(args, kwargs) -> dict:
     import re
 
     # B404:import_subprocess only used for internal Gramex scripts
-    from subprocess import check_output  # nosec B404
+    from subprocess import check_output  # noqa S404
 
     project_path = os.getcwd() if len(args) == 0 else args[0]
     project_yaml = _gramex_yaml_path(project_path, kwargs)
@@ -572,7 +572,7 @@ def walk(node: dict, parents: tuple = ()):
     try:
         app = gramex.config.PathConfig(project_yaml)
         conf = +gramex.config.ChainConfig([('base', base), ('app', app)])
-    except Exception as e:  # noqa: B902 capture load errors as a "feature"
+    except Exception as e:  # noqa B902 capture load errors as a "feature"
         app_log.exception(str(e))
         return
     yamlpaths = {'.'.join(key): val for key, val in walk(conf)}
@@ -594,7 +594,9 @@ def walk(node: dict, parents: tuple = ()):
     # Calculate JS complexity
     # B602:subprocess_popen_with_shell_equals_true and
     # B607:start_process_with_partial_path are safe to skip since this is a Gramex internal cmd
-    output = check_output('npx --yes @gramex/escomplexity', cwd=project_path, shell=True)  # nosec
+    output = check_output(
+        'npx --yes @gramex/escomplexity', cwd=project_path, shell=True  # noqa S607
+    )  # noqa S602
     es_complexity = int(output.decode('utf-8').split('\n')[-2].strip())
     return pd.DataFrame(
         {
@@ -774,11 +776,10 @@ def run_command(config):
     cygwin, cygpath, kwargs = which('cygcheck'), which('cygpath'), {'universal_newlines': True}
     if cygwin is not None and cygpath is not None:
         # subprocess.check_output is safe here since these are developer-initiated
-        # B404:import_subprocess check_output is safe here since these are developer-initiated
-        path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip()  # nosec 404
-        is_cygwin_app = check_output([cygwin, '-f', path], **kwargs).strip()  # nosec 404
+        path = check_output([cygpath, '-au', which(appcmd[0])], **kwargs).strip()  # noqa S603
+        is_cygwin_app = check_output([cygwin, '-f', path], **kwargs).strip()  # noqa S603
         if is_cygwin_app:
-            target = check_output([cygpath, '-au', target], **kwargs).strip()  # nosec 404
+            target = check_output([cygpath, '-au', target], **kwargs).strip()  # noqa S603
     # Replace TARGET with the actual target
     if 'TARGET' in appcmd:
         appcmd = [target if arg == 'TARGET' else arg for arg in appcmd]
@@ -789,7 +790,7 @@ def run_command(config):
         app_log.error(f'Cannot delete target {config.target}. Aborting installation')
         return
     # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated
-    proc = Popen(appcmd, bufsize=-1, **kwargs)  # nosec 603
+    proc = Popen(appcmd, bufsize=-1, **kwargs)  # noqa S603
     proc.communicate()
     return proc.returncode
 
@@ -873,7 +874,7 @@ def save_user_config(appname, value):
         with user_conf_file.open(encoding='utf-8') as handle:
             # If app.yaml is empty, yaml.safe_load returns None. Use the AttrDict() instead
             # B506:yaml_load is safe since this object is internally created
-            user_config = yaml.load(handle, Loader=AttrDictYAMLLoader) or user_config  # nosec
+            user_config = yaml.load(handle, Loader=AttrDictYAMLLoader) or user_config  # noqa S506
     if value is None:
         if appname in user_config:
             del user_config[appname]
@@ -884,7 +885,7 @@ def save_user_config(appname, value):
     with user_conf_file.open(mode='w', encoding='utf-8') as handle:
         # Use yaml.dump (not .safe_dump) to serialize the AttrDicts
         # B506:yaml_load is safe since this object is internally created
-        yaml.dump(user_config, handle, indent=4, default_flow_style=False)  # nosec
+        yaml.dump(user_config, handle, indent=4, default_flow_style=False)  # noqa S506
 
 
 def get_app_config(appname, kwargs):
@@ -918,7 +919,7 @@ def _check_output(cmd, default=b'', **kwargs):
     '''Run cmd and return output. Return default in case the command fails'''
     try:
         # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated
-        return check_output(shlex.split(cmd), **kwargs).strip()  # nosec B603
+        return check_output(shlex.split(cmd), **kwargs).strip()  # noqa S603
     # OSError is raised if the cmd is not found.
     # CalledProcessError is raised if the cmd returns an error.
     except (OSError, CalledProcessError):
@@ -930,7 +931,7 @@ def _run_console(cmd, **kwargs):
     cmd = shlex.split(cmd)
     try:
         # B603:subprocess_without_shell_equals_true is safe since this is developer-initiated
-        proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs)  # nosec B603
+        proc = Popen(cmd, bufsize=-1, universal_newlines=True, **kwargs)  # noqa S603
     except OSError:
         app_log.error(f'Cannot find command: {cmd[0]}')
         raise

gramex/pptgen/color.py

@@ -105,7 +105,7 @@ def __init__(self, *values):
         self.values = values
 
     def __getitem__(self, key):
-        if isinstance(key, slice) or type(key) is int:
+        if isinstance(key, (slice, int)):
             return self.values.__getitem__(key)
         elif key in self._lookup:
             return self.values[self._lookup[key]]