#284 added admin interface for script code edit/upload

samples/configs/destroy_world.json

@@ -1,6 +1,6 @@
 {
   "name": "destroy_world",
-  "script_path": "./samples/scripts/destroy_world.py",
+  "script_path": "/usr/bin/python3 ./samples/scripts/destroy_world.py",
   "description": "This is a very dangerous script, please be careful when running. Don't forget your protective helmet.",
   "requires_terminal": false,
   "output_format": "terminal"

src/auth/authorization.py

@@ -22,10 +22,11 @@ def _normalize_users(allowed_users):
 
 
 class Authorizer:
-    def __init__(self, app_allowed_users, admin_users, full_history_users, groups_provider):
+    def __init__(self, app_allowed_users, admin_users, full_history_users, code_editor_users, groups_provider):
         self._app_allowed_users = _normalize_users(app_allowed_users)
         self._admin_users = _normalize_users(admin_users)
         self._full_history_users = _normalize_users(full_history_users)
+        self._code_editor_users = _normalize_users(code_editor_users)
 
         self._groups_provider = groups_provider
 
@@ -38,6 +39,9 @@ def is_admin(self, user_id):
     def has_full_history_access(self, user_id):
         return self.is_admin(user_id) or self._is_allowed_internal(user_id, self._full_history_users)
 
+    def can_edit_code(self, user_id):
+        return self.is_admin(user_id) and self._is_allowed_internal(user_id, self._code_editor_users)
+
     def is_allowed(self, user_id, allowed_users):
         normalized_users = _normalize_users(allowed_users)
 

src/config/config_service.py

@@ -2,51 +2,65 @@
 import logging
 import os
 import re
+from collections import namedtuple
+from typing import Union
 
+from auth.authorization import Authorizer
 from config.exceptions import InvalidConfigException
 from model import script_config
 from model.model_helper import InvalidFileException
 from model.script_config import get_sorted_config
-from utils import os_utils, file_utils
+from utils import os_utils, file_utils, process_utils
 from utils.file_utils import to_filename
-from utils.string_utils import is_blank
+from utils.string_utils import is_blank, strip
+
+SCRIPT_EDIT_CODE_MODE = 'new_code'
+SCRIPT_EDIT_UPLOAD_MODE = 'upload_script'
+SCRIPT_EDIT_PATH_MODE = 'new_path'
+
+SCRIPT_PATH_FIELD = 'script_path'
+WORKING_DIR_FIELD = 'working_directory'
 
 LOGGER = logging.getLogger('config_service')
 
+ConfigSearchResult = namedtuple('ConfigSearchResult', ['short_config', 'path', 'config_object'])
+
 
 def _script_name_to_file_name(script_name):
-    escaped_whitespaces = re.sub('[\\s/]+', '_', script_name).strip("_")
-    filename = to_filename(escaped_whitespaces)
+    filename = _escape_characters_in_filename(script_name)
     return filename + '.json'
 
 
+def _escape_characters_in_filename(script_name):
+    escaped = re.sub('[\\s/]+', '_', script_name).strip("_")
+    return to_filename(escaped)
+
+
 def _preprocess_incoming_config(config):
     name = config.get('name')
     if is_blank(name):
         raise InvalidConfigException('Script name is required')
     config['name'] = name.strip()
 
-    script_path = config.get('script_path')
-    if is_blank(script_path):
-        raise InvalidConfigException('Script path is required')
-    config['script_path'] = script_path.strip()
-
 
 class ConfigService:
     def __init__(self, authorizer, conf_folder) -> None:
-        self._authorizer = authorizer
+        self._authorizer = authorizer  # type: Authorizer
         self._script_configs_folder = os.path.join(conf_folder, 'runners')
+        self._scripts_folder = os.path.join(conf_folder, 'scripts')
 
         file_utils.prepare_folder(self._script_configs_folder)
 
     def load_config(self, name, user):
         self._check_admin_access(user)
 
-        (short_config, path, config_object) = self._find_config(name)
+        search_result = self._find_config(name)
 
-        if path is None:
+        if search_result is None:
             return None
 
+        (short_config, path, config_object) = search_result
+
         if config_object.get('name') is None:
             config_object['name'] = short_config.name
 
@@ -55,45 +69,93 @@ def load_config(self, name, user):
 
         return {'config': config_object, 'filename': os.path.basename(path)}
 
-    def create_config(self, user, config):
+    def create_config(self, user, config, uploaded_script):
         self._check_admin_access(user)
         _preprocess_incoming_config(config)
 
         name = config['name']
 
-        (short_config, path, json_object) = self._find_config(name)
-        if path is not None:
+        search_result = self._find_config(name)
+        if search_result is not None:
             raise InvalidConfigException('Another config with the same name already exists')
 
+        self._preprocess_script_fields(config, None, uploaded_script, user)
+
         path = os.path.join(self._script_configs_folder, _script_name_to_file_name(name))
         unique_path = file_utils.create_unique_filename(path, 100)
 
         LOGGER.info('Creating new script config "' + name + '" in ' + unique_path)
         self._save_config(config, unique_path)
 
-    def update_config(self, user, config, filename):
+    def update_config(self, user, config, filename, uploaded_script):
         self._check_admin_access(user)
+
         _preprocess_incoming_config(config)
 
         if is_blank(filename):
             raise InvalidConfigException('Script filename should be specified')
 
         original_file_path = os.path.join(self._script_configs_folder, filename)
+
         if not os.path.exists(original_file_path):
             raise InvalidFileException(original_file_path, 'Failed to find script path: ' + original_file_path)
 
+        with open(original_file_path, 'r') as f:
+            original_config_json = json.load(f)
+            short_original_config = script_config.read_short(original_file_path, original_config_json)
+
         name = config['name']
 
-        (short_config, found_config_path, json_object) = self._find_config(name)
-        if (found_config_path is not None) and (os.path.basename(found_config_path) != filename):
+        search_result = self._find_config(name)
+        if (search_result is not None) and (os.path.basename(search_result.path) != filename):
             raise InvalidConfigException('Another script found with the same name: ' + name)
 
-        if (short_config is not None) and not self._can_edit_script(user, short_config):
-            raise ConfigNotAllowedException(str(user) + ' is not allowed to modify ' + short_config.name)
+        if not self._can_edit_script(user, short_original_config):
+            raise ConfigNotAllowedException(str(user) + ' is not allowed to modify ' + short_original_config.name)
+
+        self._preprocess_script_fields(config, original_config_json, uploaded_script, user)
 
         LOGGER.info('Updating script config "' + name + '" in ' + original_file_path)
         self._save_config(config, original_file_path)
 
+    def load_script_code(self, script_name, user):
+        if not self._authorizer.can_edit_code(user.user_id):
+            logging.warning('User ' + str(user) + ' is not allowed to edit code')
+            raise InvalidAccessException('Code edit is not allowed for this user')
+
+        config_wrapper = self.load_config(script_name, user)
+        if config_wrapper is None:
+            return None
+
+        config = config_wrapper.get('config')
+        return self._load_script_code_by_config(config)
+
+    def _load_script_code_by_config(self, plain_config):
+        script_path = plain_config.get(SCRIPT_PATH_FIELD)
+        if is_blank(script_path):
+            raise InvalidFileException('', 'Script path is not specified')
+
+        command = process_utils.split_command(script_path, plain_config.get(WORKING_DIR_FIELD))
+        binary_files = []
+        for argument in command:
+            if file_utils.exists(argument):
+                if file_utils.is_binary(argument):
+                    binary_files.append(argument)
+                    continue
+
+                return {'code': file_utils.read_file(argument), 'file_path': argument}
+
+        if binary_files:
+            if len(binary_files) == 1:
+                return {'code': None, 'file_path': binary_files[0], 'code_edit_error': 'Cannot edit binary file'}
+
+            raise InvalidFileException('command', 'Cannot choose which binary file to edit: ' + str(binary_files))
+
+        if len(command) == 1:
+            return {'code': None, 'file_path': command[0], 'code_edit_error': 'Script path does not exist'}
+
+        raise InvalidFileException('command', 'Failed to find script path in command "' + script_path + '"')
+
     def _save_config(self, config, path):
         sorted_config = get_sorted_config(config)
         config_json = json.dumps(sorted_config, indent=2)
@@ -127,15 +189,17 @@ def load_script(path, content):
         return self._visit_script_configs(load_script)
 
     def load_config_model(self, name, user, parameter_values=None, skip_invalid_parameters=False):
-        (short_config, path, json_object) = self._find_config(name)
+        search_result = self._find_config(name)
 
-        if path is None:
+        if search_result is None:
             return None
 
+        (short_config, path, config_object) = search_result
+
         if not self._can_access_script(user, short_config):
             raise ConfigNotAllowedException()
 
-        return self._load_script_config(path, json_object, user, parameter_values, skip_invalid_parameters)
+        return self._load_script_config(path, config_object, user, parameter_values, skip_invalid_parameters)
 
     def _visit_script_configs(self, visitor):
         configs_dir = self._script_configs_folder
@@ -164,7 +228,7 @@ def _visit_script_configs(self, visitor):
 
         return result
 
-    def _find_config(self, name):
+    def _find_config(self, name) -> Union[ConfigSearchResult, None]:
         def find_and_load(path, content):
             try:
                 json_object = json.loads(content)
@@ -179,15 +243,16 @@ def find_and_load(path, content):
             if short_config.name != name.strip():
                 return None
 
-            raise StopIteration((short_config, path, json_object))
+            raise StopIteration(ConfigSearchResult(short_config, path, json_object))
 
         configs = self._visit_script_configs(find_and_load)
         if not configs:
-            return None, None, None
+            return None
 
         return configs[0]
 
-    def _load_script_config(self, path, content_or_json_dict, user, parameter_values, skip_invalid_parameters):
+    @staticmethod
+    def _load_script_config(path, content_or_json_dict, user, parameter_values, skip_invalid_parameters):
         if isinstance(content_or_json_dict, str):
             json_object = json.loads(content_or_json_dict)
         else:
@@ -214,6 +279,69 @@ def _check_admin_access(self, user):
         if not self._authorizer.is_admin(user.user_id):
             raise AdminAccessRequiredException('Admin access to scripts is prohibited for ' + str(user))
 
+    def _preprocess_script_fields(self, config, original_config_json, uploaded_script, user):
+        script_config = config.get('script')
+        if not script_config:
+            raise InvalidConfigException('script option is required')
+
+        if SCRIPT_PATH_FIELD in config:
+            del config[SCRIPT_PATH_FIELD]
+        del config['script']
+
+        new_path = strip(script_config.get('path'))
+        if is_blank(new_path):
+            raise InvalidConfigException('script.path option is required')
+
+        config[SCRIPT_PATH_FIELD] = new_path
+
+        mode = script_config.get('mode')
+        if is_blank(mode) or mode == SCRIPT_EDIT_PATH_MODE:
+            pass
+
+        elif mode in (SCRIPT_EDIT_UPLOAD_MODE, SCRIPT_EDIT_CODE_MODE):
+            if not self._authorizer.can_edit_code(user.user_id):
+                raise InvalidAccessException('User ' + str(user) + ' is not allowed to edit code')
+
+            if mode == SCRIPT_EDIT_UPLOAD_MODE:
+                if uploaded_script is None:
+                    raise InvalidConfigException('Uploaded script should be specified')
+
+            if original_config_json is None:  # new config
+                if mode == SCRIPT_EDIT_UPLOAD_MODE:
+                    # escaped name is needed, when uploaded file and server has different OSes,
+                    # thus different special characters
+                    escaped_name = to_filename(uploaded_script.filename)
+                    target_path = os.path.join(self._scripts_folder, escaped_name)
+                else:
+                    filename = os.path.basename(new_path)
+                    target_path = os.path.join(self._scripts_folder, _escape_characters_in_filename(filename))
+
+                script_path = file_utils.create_unique_filename(target_path, 100)
+                config[SCRIPT_PATH_FIELD] = script_path
+
+            else:
+                existing_code = self._load_script_code_by_config(original_config_json)
+                script_path = existing_code['file_path']
+
+                if (mode == SCRIPT_EDIT_CODE_MODE) and existing_code.get('code_edit_error') is not None:
+                    raise InvalidConfigException('Failed to edit code: ' + existing_code.get('code_edit_error'))
+
+                if new_path != original_config_json.get(SCRIPT_PATH_FIELD):
+                    raise InvalidConfigException('script.path override is not allowed for ' + mode + ' mode')
+
+            if mode == SCRIPT_EDIT_UPLOAD_MODE:
+                file_utils.write_file(script_path, uploaded_script.body, byte_content=True)
+            else:
+                code = script_config.get('code')
+                if code is None:
+                    raise InvalidConfigException('script.code should be specified')
+                file_utils.write_file(script_path, code)
+
+            file_utils.make_executable(script_path)
+
+        else:
+            raise InvalidConfigException('Unsupported mode: ' + mode)
+
 
 class ConfigNotAllowedException(Exception):
     def __init__(self, message=None):
@@ -225,3 +353,6 @@ def __init__(self, message):
         super().__init__(message)
 
 
+class InvalidAccessException(Exception):
+    def __init__(self, message=None):
+        super().__init__(message)

src/main.py

@@ -91,6 +91,7 @@ def main():
         server_config.allowed_users,
         server_config.admin_users,
         server_config.full_history_users,
+        server_config.code_editor_users,
         group_provider)
 
     config_service = ConfigService(authorizer, CONFIG_FOLDER)

src/model/server_conf.py

@@ -34,6 +34,7 @@ def __init__(self) -> None:
         self.user_groups = None
         self.admin_users = []
         self.full_history_users = []
+        self.code_editor_users = []
         self.max_request_size_mb = None
         self.callbacks_config = None
         self.user_header_name = None
@@ -122,10 +123,12 @@ def from_json(conf_path, temp_folder):
         trusted_ips = strip(read_list(access_config, 'trusted_ips', default=def_trusted_ips))
         admin_users = _parse_admin_users(access_config, default_admins=def_admins)
         full_history_users = _parse_history_users(access_config)
+        code_editor_users = _parse_code_editor_users(access_config, admin_users)
     else:
         trusted_ips = def_trusted_ips
         admin_users = def_admins
         full_history_users = []
+        code_editor_users = def_admins
 
     security = model_helper.read_dict(json_object, 'security')
 
@@ -136,6 +139,7 @@ def from_json(conf_path, temp_folder):
     config.user_groups = user_groups
     config.admin_users = admin_users
     config.full_history_users = full_history_users
+    config.code_editor_users = code_editor_users
     config.user_header_name = user_header_name
     config.ip_validator = TrustedIpValidator(trusted_ips)
 
@@ -229,6 +233,15 @@ def _parse_history_users(json_object):
     return full_history_users
 
 
+def _parse_code_editor_users(json_object, admin_users):
+    full_code_editor_users = strip(read_list(json_object, 'code_editors', default=admin_users))
+    if (isinstance(full_code_editor_users, list) and '*' in full_code_editor_users) \
+            or full_code_editor_users == '*':
+        return [ANY_USER]
+
+    return full_code_editor_users
+
+
 def _parse_xsrf_protection(security):
     return model_helper.read_str_from_config(security,
                                              'xsrf_protection',