From 3c146cb3c75a015363f7a96758adf6dcc43032d6 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Mon, 28 Mar 2022 14:58:29 +0200 Subject: [PATCH] first prototype --- ck2/.gitignore | 141 ++++ ck2/CONTRIBUTING.md | 12 + ck2/LICENSE.CK | 128 ++++ ck2/README.md | 10 +- ck2/cmind/__init__.py | 6 + ck2/cmind/artifact.py | 57 ++ ck2/cmind/cli.py | 44 ++ ck2/cmind/config.py | 90 +++ ck2/cmind/kernel.py | 547 ++++++++++++++++ ck2/cmind/module.py | 383 +++++++++++ ck2/cmind/repo.py | 47 ++ ck2/cmind/repo/automation/automation/_cm.json | 10 + .../automation/cmind_automation/__init__.py | 7 + .../automation/cmind_automation/module.py | 83 +++ .../cmind_automation/module_dummy.py | 27 + .../automation/automation/requirements.txt | 1 + ck2/cmind/repo/automation/automation/setup.py | 57 ++ ck2/cmind/repo/automation/ck/_cm.json | 12 + .../repo/automation/ck/cmind_ck/__init__.py | 7 + .../repo/automation/ck/cmind_ck/module.py | 34 + ck2/cmind/repo/automation/ck/requirements.txt | 2 + ck2/cmind/repo/automation/ck/setup.py | 57 ++ ck2/cmind/repo/automation/kernel/_cm.json | 11 + .../kernel/cmind_kernel/__init__.py | 7 + .../automation/kernel/cmind_kernel/module.py | 27 + .../repo/automation/kernel/requirements.txt | 1 + ck2/cmind/repo/automation/kernel/setup.py | 57 ++ ck2/cmind/repo/automation/repo/_cm.json | 11 + .../automation/repo/cmind_repo/__init__.py | 7 + .../repo/automation/repo/cmind_repo/module.py | 495 ++++++++++++++ .../repo/automation/repo/requirements.txt | 1 + ck2/cmind/repo/automation/repo/setup.py | 57 ++ ck2/cmind/repo/cmr.yaml | 3 + ck2/cmind/repos.py | 309 +++++++++ ck2/cmind/utils.py | 619 ++++++++++++++++++ ck2/setup.py | 80 +++ 36 files changed, 3442 insertions(+), 5 deletions(-) create mode 100644 ck2/.gitignore create mode 100644 ck2/CONTRIBUTING.md create mode 100644 ck2/LICENSE.CK create mode 100644 ck2/cmind/__init__.py create mode 100644 ck2/cmind/artifact.py create mode 100644 ck2/cmind/cli.py create mode 100644 ck2/cmind/config.py create mode 100644 ck2/cmind/kernel.py create mode 100644 ck2/cmind/module.py create mode 100644 ck2/cmind/repo.py create mode 100644 ck2/cmind/repo/automation/automation/_cm.json create mode 100644 ck2/cmind/repo/automation/automation/cmind_automation/__init__.py create mode 100644 ck2/cmind/repo/automation/automation/cmind_automation/module.py create mode 100644 ck2/cmind/repo/automation/automation/cmind_automation/module_dummy.py create mode 100644 ck2/cmind/repo/automation/automation/requirements.txt create mode 100644 ck2/cmind/repo/automation/automation/setup.py create mode 100644 ck2/cmind/repo/automation/ck/_cm.json create mode 100644 ck2/cmind/repo/automation/ck/cmind_ck/__init__.py create mode 100644 ck2/cmind/repo/automation/ck/cmind_ck/module.py create mode 100644 ck2/cmind/repo/automation/ck/requirements.txt create mode 100644 ck2/cmind/repo/automation/ck/setup.py create mode 100644 ck2/cmind/repo/automation/kernel/_cm.json create mode 100644 ck2/cmind/repo/automation/kernel/cmind_kernel/__init__.py create mode 100644 ck2/cmind/repo/automation/kernel/cmind_kernel/module.py create mode 100644 ck2/cmind/repo/automation/kernel/requirements.txt create mode 100644 ck2/cmind/repo/automation/kernel/setup.py create mode 100644 ck2/cmind/repo/automation/repo/_cm.json create mode 100644 ck2/cmind/repo/automation/repo/cmind_repo/__init__.py create mode 100644 ck2/cmind/repo/automation/repo/cmind_repo/module.py create mode 100644 ck2/cmind/repo/automation/repo/requirements.txt create mode 100644 ck2/cmind/repo/automation/repo/setup.py create mode 100644 ck2/cmind/repo/cmr.yaml create mode 100644 ck2/cmind/repos.py create mode 100644 ck2/cmind/utils.py create mode 100644 ck2/setup.py diff --git a/ck2/.gitignore b/ck2/.gitignore new file mode 100644 index 0000000000..2fe08ed8ea --- /dev/null +++ b/ck2/.gitignore @@ -0,0 +1,141 @@ +# Misc +build +build/* +MANIFEST +*.pyc +*tmp/ +__pycache__ +.cache/ +.coverage +htmlcov + + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/ck2/CONTRIBUTING.md b/ck2/CONTRIBUTING.md new file mode 100644 index 0000000000..ac224dca4a --- /dev/null +++ b/ck2/CONTRIBUTING.md @@ -0,0 +1,12 @@ +# Contributing to CM (CK2) + +* Pull requests: TBA +* Issues: TBA +* Mailing list: TBA +* Discord channel: TBA +* Slack channel: TBA + +# Contributors + +* Grigori Fursin, OctoML +* Thierry Moreau, OctoML diff --git a/ck2/LICENSE.CK b/ck2/LICENSE.CK new file mode 100644 index 0000000000..34f9dd94a4 --- /dev/null +++ b/ck2/LICENSE.CK @@ -0,0 +1,128 @@ +MLCOMMONS ASSOCIATION LICENSE + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +This license reproduces without alteration the terms of the Apache License + +Version 2.0, January 2004 + +http://www.apache.org/licenses/ + +1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by +Sections 1 through 9 of this document. +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is +granting the License. +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are +controlled by, or are under common control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the direction or management of such +entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this +License. +"Source" form shall mean the preferred form for making modifications, including but not limited to +software source code, documentation source, and configuration files. +"Object" form shall mean any form resulting from mechanical transformation or translation of a +Source form, including but not limited to compiled object code, generated documentation, and +conversions to other media types. +"Work" shall mean the work of authorship, whether in Source or Object form, made available under +the License, as indicated by a copyright notice that is included in or attached to the work (an example +is provided in the Appendix below). +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or +derived from) the Work and for which the editorial revisions, annotations, elaborations, or other +modifications represent, as a whole, an original work of authorship. For the purposes of this License, +Derivative Works shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. +"Contribution" shall mean any work of authorship, including the original version of the Work and +any modifications or additions to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal +Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent to the Licensor or +its representatives, including but not limited to communication on electronic mailing lists, source +code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor +for the purpose of discussing and improving the Work, but excluding communication that is +conspicuously marked or otherwise designated in writing by the copyright owner as "Not a +Contribution." +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a +Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each +Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +1106217.1 +irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly +perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor +hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, +and otherwise transfer the Work, where such license applies only to those patent claims licensable by +such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of +their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute +patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging +that the Work or a Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License for that Work shall +terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works +thereof in any medium, with or without modifications, and in Source or Object form, provided that +You meet the following conditions: + + 1. You must give any other recipients of the Work or Derivative Works a copy of this License; +and + + 2. You must cause any modified files to carry prominent notices stating that You changed the +files; and + + 3. You must retain, in the Source form of any Derivative Works that You distribute, all +copyright, patent, trademark, and attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of the Derivative Works; and + + 4. If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative +Works that You distribute must include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not pertain to any part of the +Derivative Works, in at least one of the following places: within a NOTICE text file +distributed as part of the Derivative Works; within the Source form or documentation, if +provided along with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of the +NOTICE file are for informational purposes only and do not modify the License. You may +add Your own attribution notices within Derivative Works that You distribute, alongside or +as an addendum to the NOTICE text from the Work, provided that such additional +attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or distribution of +Your modifications, or for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with the conditions stated in +this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution +intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms +and conditions of this License, without any additional terms or conditions. Notwithstanding the +above, nothing herein shall supersede or modify the terms of any separate license agreement you +may have executed with Licensor regarding such Contributions. +6. Trademarks. This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for reasonable and customary +use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor +provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, +without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, +MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for +determining the appropriateness of using or redistributing the Work and assume any risks associated +with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including +negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including +any direct, indirect, special, incidental, or consequential damages of any character arising as a result +of this License or out of the use or inability to use the Work (including but not limited to damages for +loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial +damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative +Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, +indemnity, or other liability obligations and/or rights consistent with this License. However, in +accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not +on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each +Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by +reason of your accepting any such warranty or additional liability. diff --git a/ck2/README.md b/ck2/README.md index 3aef8675bf..d01487d3a7 100644 --- a/ck2/README.md +++ b/ck2/README.md @@ -1,6 +1,6 @@ -# Collective Mind framework aka as CK2 or CM +# Collective Mind automation framework (CM or CK2) -*Following the success of the [Collective Knowledge framework (CK)](https://github.com/mlcommons/ck) +*Following the success of the [Collective Knowledge automation framework (CK)](https://github.com/mlcommons/ck) to automate [MLPerf benchmark](https://github.com/mlcommons/ck/tree/master/docs/mlperf-automation), simplify the deployment of optimized AI/ML systems [in production](https://cKnowledge.org/partners.html) and enable [reproducible experiments](https://cTuning.org/ae), @@ -35,7 +35,7 @@ scientific experimentation, FAIR principles, and best practices for reproducible # Related community events -* **2022 April:** HPCA'22 reproducibility talk +* **2022 April 3:** HPCA'22 talk about using CK/CK2 to automate MLPerf design space exploration and deploy efficient AI/ML Systems * **2022 March:** Presentation about our experience connecting MLOps and DevOps @@ -46,10 +46,10 @@ scientific experimentation, FAIR principles, and best practices for reproducible # Feedback -The CM framework is being developed with the community and based on the community feedback! +The CM framework (CK2) is being developed with the community and based on the community feedback! Please don't hesitate to get in touch with us if you have any suggestions about how to bridge the growing gap between scientific research and production. -Please report encountered issues [here](https://github.com/octoml/cm/issues). +Please report encountered issues [here](https://github.com/mlcommons/ck/issues). # Coordinators diff --git a/ck2/cmind/__init__.py b/ck2/cmind/__init__.py new file mode 100644 index 0000000000..5a89da75a4 --- /dev/null +++ b/ck2/cmind/__init__.py @@ -0,0 +1,6 @@ +__version__ = "0.5.1" + +from cmind.kernel import access +from cmind.kernel import error +from cmind.kernel import halt +from cmind.kernel import CM diff --git a/ck2/cmind/artifact.py b/ck2/cmind/artifact.py new file mode 100644 index 0000000000..1d4557b46f --- /dev/null +++ b/ck2/cmind/artifact.py @@ -0,0 +1,57 @@ +# Collective Mind artifact + +import os + +from cmind import utils + +class Artifact: + ############################################################ + def __init__(self, cmind, path): + """ + Initialize artifact class + """ + + self.cmind = cmind + + self.cfg = cmind.cfg + + self.path = path + self.meta = {} + + ############################################################ + def load(self, ignore_inheritance = False, base_recursion = 0): + """ + Load artifact + + """ + + path_artifact_meta = os.path.join(self.path, self.cfg['file_cmeta']) + + r = utils.is_file_json_or_yaml(path_artifact_meta) + if r['return'] >0 : return r + + if not r['is_file']: + return {'return':16, 'error': 'CM artifact not found in path {}'.format(self.path)} + + # Search if there is a repo in this path + r = utils.load_yaml_and_json(file_name_without_ext = path_artifact_meta) + if r['return'] >0: return r + + meta = r['meta'] + + if not ignore_inheritance: + automation_uid = meta.get('automation_uid', '') + automation_alias = meta.get('automation_alias', '') + automation = automation_alias + if automation_uid!='': automation+=','+automation_uid + + # Check inheritance + r = utils.process_meta_for_inheritance({'automation':automation, + 'meta':meta, + 'cmind':self.cmind, + 'base_recursion':base_recursion}) + if r['return']>0: return r + + self.meta = r['meta'] + + return {'return':0} diff --git a/ck2/cmind/cli.py b/ck2/cmind/cli.py new file mode 100644 index 0000000000..1b5e9e4ea1 --- /dev/null +++ b/ck2/cmind/cli.py @@ -0,0 +1,44 @@ +# Collective Mind command line wrapper + +import sys + +def run(argv = None): + """ + Run CM from command line. + + Args: + argv (str | list): CM input + + Returns: + Dictionary: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + data from a given action + """ + + # Access CM + from cmind.kernel import CM + + cm = CM(out='con') + + if argv is None: + argv = sys.argv[1:] + + r = cm.access(argv) + + ret = r['return'] + + # Print error only in CLI + if ret > 0: + # Process error: either raise or just print to stderr + # depending on settings + cm.error(r) + + sys.exit(ret) + + +if __name__ == "__main__": + run() diff --git a/ck2/cmind/config.py b/ck2/cmind/config.py new file mode 100644 index 0000000000..1181ae02ad --- /dev/null +++ b/ck2/cmind/config.py @@ -0,0 +1,90 @@ +# Collective Mind configuration + +import os + +class Config(object): + """ + CM configuration class + """ + + def __init__(self, config_file: str = None): + """ + Initialize Collective Mind configuration + """ + + self.cfg = { + "name": "cm", + + "env_home": "CM_HOME", + "env_debug": "CM_DEBUG", + "env_kernel": "CM_KERNEL", + + "flag_debug": "cm-debug", + + "flag_out": "out", + + "flag_help": "h", + "flag_help2": "help", + + "error_prefix": "CM error:", + "info_cli": "cm {automation} {action} {artifact(s)} {flags}", + + "default_home_dir": "CM", + + "file_repos": "repos.json", + "dir_repos": "repos", + "cmind_repo": "repo", + + "file_meta_repo": "cmr", + + "cmind_python_module_prefix": "cmind_", + "cmind_python_module_uid": "cmind_uid", + + "action_substitutions": { + "ls":"search", + "list":"search", + "find":"search", + "rm":"delete" + }, + + "action_substitutions_reverse": { + "search":["list", "ls", "find"], + "delete":["rm"] + }, + + "cmind_automation":"automation", + + "file_cmeta":"_cm", + + "local_repo_name": "local", + "local_repo_meta": { + "uid": "9a3280b14a4285c9", + "alias": "local", + "name": "local CM repository" + }, + + "default_repo_name": "default", + + "default_repo_pack": "cm.zip", + + "repo_url_prefix":"https://github.com/", + "repo_url_org":"mlcommons", + + "line":"=======================================================" + } + + # Attempt to update config from file if specified explicitly during initialization + # or specified by the environment variable + if config_file is None or config_file.strip() == '': + config_file = os.environ.get(self.cfg['env_kernel']) + + if config_file is not None and config_file.strip() != '': + from cmind import utils + r = utils.load_json_or_yaml(config_file) + if r['return'] > 0: + # Raise here because it's an initializer of a class + raise Exception(r['error']) + + meta = r.get('meta', {}) + + self.cfg.update(meta) diff --git a/ck2/cmind/kernel.py b/ck2/cmind/kernel.py new file mode 100644 index 0000000000..3e4d661a58 --- /dev/null +++ b/ck2/cmind/kernel.py @@ -0,0 +1,547 @@ +# Collective Mind kernel functions + +from cmind.config import Config +from cmind.repos import Repos +from cmind.module import Module +from cmind import utils + +import sys +import os +import imp +import importlib +import pkgutil +import inspect + +cm = None + +############################################################ +def access(i): + """ + Initialize and access Collective Mind without customization + """ + + global cm + + if cm is None: + cm=CM() + + return cm.access(i) + +############################################################ +def error(i): + """ + Print error + """ + + # Force console + con_tmp = cm.con + + cm.con=True + + r = cm.error(i) + + cm.con = con_tmp + +############################################################ +def halt(i): + """ + Print error and halt + """ + + return cm.halt(i) + +############################################################ +class CM(object): + """ + Main Collective Mind class + """ + + ############################################################ + def __init__(self, home_path = '', debug = False, out = ''): + """ + Initialize Collective Mind class + """ + +# print ('************************************************') +# print ('Initialize CM ...') +# print ('************************************************') + + # Check output (for now support con but can be json or file, etc) + self.out = out + self.con = False + if out == 'con': + self.con = True + + # Initialize and update CM configuration + self.cfg = Config().cfg + + # Check if debug (raise error instead of soft error) + self.debug = False + if debug or os.environ.get(self.cfg['env_debug'],'').strip().lower()=='yes': + self.debug = True + + # Explicit path to direcory with CM repositories and other internals + self.home_path = home_path + if self.home_path == '': + s = os.environ.get(self.cfg['env_home'],'').strip() + if s != '': + self.home_path = s + + if self.home_path == '': + from os.path import expanduser + self.home_path = os.path.join(expanduser("~"), self.cfg['default_home_dir']) + + self.path_to_cmind_kernel = inspect.getfile(inspect.currentframe()) + self.path_to_cmind = os.path.dirname(self.path_to_cmind_kernel) + self.path_to_cmind_repo = os.path.join(self.path_to_cmind, self.cfg['cmind_repo']) + + # Repositories + self.repos = None + + # Default module + self.default_module = None + + ############################################################ + def parse_cli(self, cmd): + """ + Parse CM command line. + + Args: + cmd (str | list) : arguments as a string or list + + Returns: + Dictionary: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + cm_input (dict): CM input + """ + + # If input is string, convert to argv + # We use shlex to properly convert "" + if cmd is None: + argv = [] + + elif type(cmd) == str: + import shlex + argv = shlex.split(cmd) + + else: + argv = cmd + + # Positional arguments + cm_input = {} + + # First argument: automation + special_cli_characters=['-', '@'] + + for key in ['automation', 'action']: + if len(argv) > 0 and argv[0].strip()[0] not in special_cli_characters: + cm_input[key] = argv.pop(0) + + # Check if just one artifact or multiple ones + artifact='' + artifacts=[] # Only added if more than 1 artifact! + + for a in argv: + if a.startswith('@'): + # Load JSON or YAML file + from cmind.utils import io + r = utils.load_json_or_yaml(file_name = a[1:], check_if_exists=True) + if r['return'] >0 : return r + + meta = r['meta'] + + cm_input.update(meta) + + elif not a.startswith('-'): + # artifact + if artifact=='': + artifact=a + cm_input['artifact']=a + + artifacts.append(a) + else: + # flags + if '=' in a: + key,value = a.split('=') + value=value.strip() + else: + key=a + value=True + + if key.startswith('-'): key=key[1:] + if key.startswith('-'): key=key[1:] + + if key.endswith(','): + key = key[:-1] + value = value.split(',') if value!="" else [] + + if '.' in key: + keys = key.split('.') + new_cm_input = cm_input + for key in keys[:-1]: + if key not in new_cm_input: + new_cm_input[key] = {} + new_cm_input = new_cm_input[key] + + new_cm_input[keys[-1]]=value + else: + cm_input[key] = value + + # Add artifacts if > 1 + if len(artifacts) > 1: + cm_input['artifacts'] = artifacts + + return {'return':0, 'cm_input':cm_input} + + ############################################################ + def error(self, r): + """ + Process CM error (print or raise and exit) + + Args: + r (dict) - output from CM function + debug (Boolean) - if True, call raise + + Returns: + None or raise + + """ + + import os + + if r['return']>0: + if self.debug: + raise Exception(r['error']) + + if self.con: + sys.stderr.write(self.cfg['error_prefix']+' '+r['error']+'\n') + + return r + + ############################################################ + def halt(self, r): + """ + Process CM error and halt (useful for scripts) + + Args: + r (dict) - output from CM function + debug (Boolean) - if True, call raise + + Returns: + None or raise + + """ + + # Force console + self.con=True + + self.error(r) + + sys.exit(r['return']) + + ############################################################ + def access(self, i): + """ + Access customized Collective Mind object + + Args: + i (dict | str | argv): CM input + + Returns: + Dictionary: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + data from a given action + + """ + + # Pass to internal access + r = self.internal_access(i) + + return r + + + ############################################################ + def internal_access(self, i): + """ + Access customized Collective Mind object + + Args: + i (dict | str | argv): CM input + + default (bool) - if True, call default automation without any specialization + (usually for pure CM database actions) + + Returns: + Dictionary: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + data from a given action + + """ + + # Check the type of input + if i is None: + i = {} + + # Attempt to detect debug flag early (though suggest to use environment) + # If error in parse_cli, it will raise error + if self.cfg['flag_debug'] in i: + self.debug = True + + # Extra parse if string or list + if type(i) == str or type(i) == list: + r = self.parse_cli(i) + if r['return'] >0 : return r + + i = r['cm_input'] + + # Check CM flag for CM output + if self.cfg['flag_out'] in i: + self.out = i[self.cfg['flag_out']] + + if self.out == 'con': + self.con = True + + # Check if has help flag + cm_help = i.get(self.cfg['flag_help'], False) or i.get(self.cfg['flag_help2'], False) + + # Initialized default module + if self.default_module == None: + self.default_module = Module(self, __file__) + + # Check automation + automation = i.get('automation','') + if automation == '': automation = i.get('a','') + + if automation == '': + if self.con: + print (self.cfg['info_cli']) + + if cm_help: + print ('') + print ('Collective database actions:') + print ('') + + for d in sorted(dir(self.default_module)): + if not d.startswith('_') and d not in ['cmind']: + print ('* '+d) + + return {'return':0, 'help':'help about database actions'} + + # Check if general help + r = utils.parse_cm_object(automation) + if r['return'] >0 : return r + + # A list of CM objects + parsed_automation = r['cm_object'] + + i['parsed_automation'] = parsed_automation + + # First object in a list is an automation + # Second optional object in a list is a repo + auto_name = parsed_automation[0] if len(parsed_automation)>0 else ('','') + auto_repo = parsed_automation[1] if len(parsed_automation)>1 else None + + artifacts = i.get('artifacts',[]) + if len(artifacts)>0: + parsed_artifacts = [] + + for artifact in artifacts: + # Parse artifact + r = utils.parse_cm_object(artifact) + if r['return'] >0 : return r + + parsed_artifacts.append(r['cm_object']) + + i['parsed_artifacts'] = parsed_artifacts + + # Check artifact and artifacts + artifact = i.get('artifact','') + if artifact != '': + # Parse artifact + r = utils.parse_cm_object(artifact) + if r['return'] >0 : return r + + i['parsed_artifact'] = r['cm_object'] + + # Load repositories + if self.repos == None: + repos = Repos(path = self.home_path, cfg = self.cfg, + path_to_default_repo = self.path_to_cmind_repo) + + r = repos.load() + if r['return'] >0 : return r + + # Set only after all initializations + self.repos = repos + + # Find automation + # TBD: need to move order to configuration + + # Search for automation in CM repositories + pruned_automations = [] + + default_automation = True if i.get('default',False) else False + + use_any_func = False + + if not default_automation: + # Search for installed automation modules + for installed_module in pkgutil.iter_modules(): + module_name = installed_module[1] + if module_name.startswith(self.cfg['cmind_python_module_prefix']): + # Check meta + path_module = installed_module[0].path + path_module_meta = os.path.join(path_module, self.cfg['file_cmeta']) + + r = utils.load_yaml_and_json(file_name_without_ext = path_module_meta) + if r['return'] >0: return r + + module_meta = r['meta'] + + installed_module_uid = module_meta['uid'] + installed_module_alias = module_meta['alias'] + + r = utils.match_objects(uid = installed_module_uid, + alias = installed_module_alias, + uid2 = auto_name[1], + alias2 = auto_name[0]) + if r['return']>0: return r + + if r['match']: + pruned_automations.append({'path':path_module, 'name':module_name, 'meta':module_meta}) + + # Search for automation module in repos (default, local, other) + r = self.default_module.search({'parsed_automation':[('automation','bbeb15d8f0a944a4')], + 'parsed_artifact':parsed_automation, + 'skip_con':True}) + if r['return']>0: return r + module_lst = r['list'] + + for module in module_lst: + pruned_automations.append({'path':module.path, 'name':module.meta['module_name'], 'meta':module.meta}) + + if len(pruned_automations)==1: + module_path = pruned_automations[0]['path'] + module_name = pruned_automations[0]['name'] + module_meta = pruned_automations[0]['meta'] + + use_any_func = module_meta.get('use_any_func',False) + + # Find module + try: + found_module = imp.find_module(module_name, [module_path]) + except ImportError as e: # pragma: no cover + return {'return': 1, 'error': 'can\'t find module code (path={}, name={}, err={})'.format(module_path, module_name, format(e))} + + module_handler = found_module[0] + module_full_path = found_module[1] + + # Generate uid for the run-time extension of the loaded module + # otherwise modules with the same extension (key.py for example) + # will be reloaded ... + + r = utils.gen_uid() + if r['return'] > 0: return r + module_run_time_uid = 'rt-'+r['uid'] + + try: + loaded_module = imp.load_module(module_run_time_uid, module_handler, module_full_path, found_module[2]) + except ImportError as e: # pragma: no cover + return {'return': 1, 'error': 'can\'t load module code (path={}, name={}, err={})'.format(module_path, module_name, format(e))} + + if module_handler is not None: + module_handler.close() + + elif len(pruned_automations)>1: + return {'return':2, 'error':'ambiguity because several modules were found for {}: {}'.format(auto_name,pruned_automations)} + + # Report an error if a repo is specified for a given automation action but it's not found there + if len(pruned_automations)==0 and auto_repo is not None: + return {'return':3, 'error':'automation is not found in a specified repo {}'.format(auto_repo)} + + if default_automation or len(pruned_automations)==0: + # TBD: work for basic functions even if module is not installed + # Maybe should be something else (internal keyword that can't be used) + auto=('module','087bf3c4403b9573') + from . import module as loaded_module + + r = loaded_module.init(self) + if r['return']>0: return r + + initialized_module = r['module'] + + # Check action + action = i.get('action','') + + if action == '': + if self.con: + print ('') + print ('Collective database actions:') + print ('') + + database_actions=dir(self.default_module) + + for d in sorted(database_actions): + if not d.startswith('_') and d not in ['module_path', 'path', 'cmind']: + x = '* '+d + + if d in self.cfg['action_substitutions_reverse']: + x += ' ('+', '.join(self.cfg['action_substitutions_reverse'][d])+')' + + print (x) + + print ('') + print ('Automation actions:') + print ('') + + for d in sorted(dir(initialized_module)): + if not d.startswith('_') and d not in ['cmind', 'meta'] and d not in database_actions: + print ('* '+d) + + return {'return':0, 'help':''} + + # Convert action into function (substitute internal words) + func = action.replace('-','_') + + if func in self.cfg['action_substitutions']: + func = self.cfg['action_substitutions'][func] + + # Check func in a class when importing + if use_any_func: + func = 'any' + + if not hasattr(initialized_module, func): + return {'return':4, 'error':'action "{}" not found in automation {}'.format(func, auto_name)} + + # Check if help for automation + if cm_help: + print ('') + print ('Action help:') + print ('') + + import inspect + path_to_module = inspect.getfile(inspect.getmodule(initialized_module)) + print ('(path: {})'.format(path_to_module)) + + print ('') + print ('TBD') + + return {'return':0} + + # Call automation action + func_addr=getattr(initialized_module, func) + r = func_addr(i) + + return r diff --git a/ck2/cmind/module.py b/ck2/cmind/module.py new file mode 100644 index 0000000000..083f2720e3 --- /dev/null +++ b/ck2/cmind/module.py @@ -0,0 +1,383 @@ +# Collective Mind module + +import os + +from cmind import utils +from cmind.artifact import Artifact + +def init(cmind): + + module = Module(cmind, __file__) + + module.meta = {'alias':'module', + 'uid':'087bf3c4403b9573'} + + return {'return':0, 'module':module} + + +class Module: + ############################################################ + def __init__(self, cmind, module_file): + """ + Initialize class and reuse initialized cmind class + """ + + self.cmind = cmind + + self.module_path = module_file + self.path = os.path.dirname(self.module_path) + + ############################################################ + def search(self, i): + """ + List CM artifacts + (simple and slow implementation - can be considerably accelerated with on-the-fly indexing as in CK) + + Args: + i (dict): + (parsed_automation) - tuple: automation (alias,UID) (repo (alias,UID)) + + (ignore_inheritance) - if True, ignore inheritance and just load meta to match UID/alias + + Returns: + Dictionary: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + """ + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + lst = [] + + # Check tags + tags = utils.get_list_from_cli(i, key='tags') + + ignore_inheritance = i.get('ignore_inheritance',False) == True + + # Get parsed automation + # First object in a list is an automation + # Second optional object in a list is a repo + parsed_automation = i.get('parsed_automation',[]) + + auto_name = parsed_automation[0] if len(parsed_automation)>0 else ('','') + + # Get parsed artifact + parsed_artifact = i.get('parsed_artifact',[]) + + artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') + artifact_repo = parsed_artifact[1] if len(parsed_artifact)>1 else None + + artifact_repo_wildcards = False + if artifact_repo is not None: + if '*' in artifact_repo[0] or '?' in artifact_repo[0]: + artifact_repo_wildcards = True + + # Prune repos if needed (check wildcards too) + repos = self.cmind.repos.lst + pruned_repos = [] + + for repo in repos: + add_repo = True + + if artifact_repo is not None: + repo_uid = repo.meta['uid'] + if repo_uid == '': repo_uid = None + + repo_alias = repo.meta['alias'] + if repo_alias == '': repo_alias = None + + r = utils.match_objects(uid = repo_uid, + alias = repo_alias, + uid2 = artifact_repo[1], + alias2 = artifact_repo[0]) + if r['return']>0: return r + + add_repo = r['match'] + + if add_repo: + pruned_repos.append(repo) + + # Iterate over pruned repositories + for repo in pruned_repos: + path_repo = repo.path_with_prefix + + automations = os.listdir(path_repo) + + for automation in automations: + path_automations = os.path.join(path_repo, automation) + + if os.path.isdir(path_automations): + # Pruning by automation + # Will need to get first entry to get UID and alias of the automation + + path_artifacts = os.path.join(path_repo, path_automations) + + artifacts = os.listdir(path_artifacts) + + # Check if artifact is first to get meta about automation UID/alias + first_artifact = True + + for artifact in artifacts: + path_artifact = os.path.join(path_artifacts, artifact) + + if os.path.isdir(path_artifact): + # Check if has CM meta to make sure that it's a CM object + path_artifact_meta = os.path.join(path_artifact, self.cmind.cfg['file_cmeta']) + + r = utils.is_file_json_or_yaml(file_name = path_artifact_meta) + if r['return']>0: return r + + if r['is_file']: + # Load artifact class + artifact_object = Artifact(cmind = self.cmind, path = path_artifact) + + r = artifact_object.load(ignore_inheritance = ignore_inheritance) + if r['return']>0: return r + + meta = artifact_object.meta + + uid = meta.get('uid', '') + if uid==None: uid = '' + + alias = meta.get('alias', '') + if alias==None: alias = '' + + if first_artifact: + # Need to check if automation matches + first_artifact = False + + automation_uid = meta.get('automation_uid') + automation_alias = meta.get('automation_alias') + + r = utils.match_objects(uid = automation_uid, + alias = automation_alias, + uid2 = auto_name[1], + alias2 = auto_name[0]) + if r['return']>0: return r + + if not r['match']: + break + + # Match object +# print ("'{}','{}','{}','{}'".format(uid,alias,artifact_obj[1],artifact_obj[0])) + r = utils.match_objects(uid = uid, + alias = alias, + uid2 = artifact_obj[1], + alias2 = artifact_obj[0]) + if r['return']>0: return r + + if r['match']: + + if len(tags)>0: + tags_in_meta = meta.get('tags',[]) + + if not all(t in tags_in_meta for t in tags): + continue + + lst.append(artifact_object) + + + + # Output paths to console if CLI or forced + if con: + for l in lst: + print (l.path) + + return {'return':0, 'list':lst} + + + ############################################################ + def add(self, i): + """ + Add Collective Mind object + + parsed_automation + parsed_artifact + meta + tags + skip_con + + + """ + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + # Get parsed automation + parsed_automation = i.get('parsed_automation',[]) + + auto_name = parsed_automation[0] if len(parsed_automation)>0 else ('','') + + # Get parsed artifact + parsed_artifact = i.get('parsed_artifact',[]) + + artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') + artifact_repo = parsed_artifact[1] if len(parsed_artifact)>1 else None + + # If artifact repo is not defined, use "local" (like scratchpad) + if artifact_repo == None: + artifact_repo = (self.cmind.cfg['local_repo_meta']['alias'], + self.cmind.cfg['local_repo_meta']['uid']) + + # Find repository + r = self.cmind.access({'automation':'repo', + 'action':'find', + 'artifact':artifact_repo[0]+','+artifact_repo[1], + 'skip_con':True}) + if r['return']>0: return r + + lst = r['lst'] + + if len(lst)==0: + return {'return':16, 'error':'repository {} not found'.format(artifact_repo)} + + if len(lst)>1: + return {'return':1, 'error':'more than 1 repository found'} + + repo = lst[0] + + repo_path = repo.path_with_prefix + + # Check automation (if exists) + automation_path = os.path.join(repo_path, auto_name[0]) if auto_name[0]!='' else os.path.join(repo_path, auto_name[1]) + + if not os.path.isdir(automation_path): + os.makedirs(automation_path) + + # Check object UID + if artifact_obj[1]=='' or artifact_obj[1] is None: + r=utils.gen_uid() + if r['return']>0: return r + + artifact_obj=(artifact_obj[0],r['uid']) + + obj_path = os.path.join(automation_path, artifact_obj[0]) if artifact_obj[0]!='' else os.path.join(automation_path, artifact_obj[1]) + + meta_path_yaml = os.path.join(obj_path, self.cmind.cfg['file_cmeta']+'.yaml') + meta_path_json = os.path.join(obj_path, self.cmind.cfg['file_cmeta']+'.json') + + # Check if artifact meta exists - then object already exists + if os.path.isfile(meta_path_yaml) or os.path.isfile(meta_path_json): + return {'return':8, 'error':'artifact already exists in the path {}'.format(obj_path)} + + if not os.path.isdir(obj_path): + os.makedirs(obj_path) + + # Prepare meta + meta = i.get('meta',{}) + tags = utils.get_list_from_cli(i, key='tags') + + if meta.get('alias','')=='': meta['alias']=artifact_obj[0] + if meta.get('uid','')=='': meta['uid']=artifact_obj[1] + if meta.get('automation_alias','')=='': meta['automation_alias']=auto_name[0] + if meta.get('automation_uid','')=='': meta['automation_uid']=auto_name[1] + + existing_tags = meta.get('tags',[]) + if len(tags)>0: + existing_tags.append(tags) + meta['tags']=existing_tags + + # Record meta + r = utils.save_json(meta_path_json, meta=meta) + if r['return']>0: return r + + if con: + print ('Added CM object at {}'.format(obj_path)) + + return {'return':0, 'meta':meta, 'path':obj_path} + + ############################################################ + def delete(self, i): + """ + Delete Collective Mind artifact + + parsed_automation + parsed_artifact + skip_con + f or force (bool) - if True do not ask for confirmation + + + """ + + import shutil + + con = True if self.cmind.con and not i.get('skip_con', False) else False + force = True if i.get('f', False) or i.get('force',False) else False + + # Find an object + i['skip_con']=True + + r = self.search(i) + if r['return']>0: return r + + lst = r['list'] + if len(lst)==0: + return {'return':16, 'error':'artifact(s) not found'} + + deleted_lst = [] + + for artifact in lst: + path_to_artifact = artifact.path + + if con: + print ('Deleting CM artifact in {} ...'.format(path_to_artifact)) + + if not force: + ask = input(' Are you sure you want to delete this artifact (y/N): ') + ask = ask.strip().lower() + + if ask!='y': + print (' Skipped!') + continue + elif not force: + # If not console mode skip unless forced + continue + + # Deleting artifact + deleted_lst.append(artifact) + + shutil.rmtree(path_to_artifact) + + if con: + print (' Deleted!') + + return {'return':0, 'deleted_list':deleted_lst} + + ############################################################ + def load(self, i): + """ + Load Collective Mind artifact + + parsed_automation + parsed_artifact + skip_con + + """ + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + # Find an object + i['skip_con']=True + + r = self.search(i) + if r['return']>0: return r + + lst = r['list'] + if len(lst)==0: + return {'return':16, 'error':'artifact not found: {}'.format(i)} + elif len(lst)>1: + return {'return':1, 'error':'more than 1 artifact found', 'lst':lst} + + artifact = lst[0] + + path = artifact.path + meta = artifact.meta + + # Output if con + if con: + import json + print (json.dumps(meta, indent=2, sort_keys=True)) + + return {'return':0, 'path':path, 'meta':meta, 'artifact':artifact} diff --git a/ck2/cmind/repo.py b/ck2/cmind/repo.py new file mode 100644 index 0000000000..0fb98db0b7 --- /dev/null +++ b/ck2/cmind/repo.py @@ -0,0 +1,47 @@ +# Collective Mind repository + +import os + +from cmind import utils + +class Repo: + ############################################################ + def __init__(self, path, cfg): + """ + Initialize class + """ + + self.cfg = cfg + + self.path = path + self.path_with_prefix = path + + self.path_prefix = '' + + # Repo meta + self.meta = {} + + ############################################################ + def load(self): + """ + Load repository file + + """ + + # Check if home directory exists. Create it otherwise. + if not os.path.isdir(self.path): + return {'return':1, 'error': 'repository path {} not found'.format(self.path)} + + # Search if there is a repo in this path + full_path = os.path.join(self.path, self.cfg['file_meta_repo']) + + r = utils.load_yaml_and_json(file_name_without_ext = full_path) + if r['return'] >0: return r + + self.meta = r['meta'] + + self.path_prefix = self.meta.get('prefix','') + if self.path_prefix !='': + self.path_with_prefix = os.path.join(self.path, self.path_prefix) + + return {'return':0} diff --git a/ck2/cmind/repo/automation/automation/_cm.json b/ck2/cmind/repo/automation/automation/_cm.json new file mode 100644 index 0000000000..6fc1a1debf --- /dev/null +++ b/ck2/cmind/repo/automation/automation/_cm.json @@ -0,0 +1,10 @@ +{ + "uid":"bbeb15d8f0a944a4", + "alias":"automation", + "automation_uid": "bbeb15d8f0a944a4", + "automation_alias": "automation", + "module_name":"cmind_automation", + "tags": [ + "automation" + ] +} diff --git a/ck2/cmind/repo/automation/automation/cmind_automation/__init__.py b/ck2/cmind/repo/automation/automation/cmind_automation/__init__.py new file mode 100644 index 0000000000..e6a7f8fd52 --- /dev/null +++ b/ck2/cmind/repo/automation/automation/cmind_automation/__init__.py @@ -0,0 +1,7 @@ +from cmind import utils + +def init(cmind): + + from .module import CModule + + return utils.init_module(CModule, cmind, __file__) diff --git a/ck2/cmind/repo/automation/automation/cmind_automation/module.py b/ck2/cmind/repo/automation/automation/cmind_automation/module.py new file mode 100644 index 0000000000..5c6ce0c06a --- /dev/null +++ b/ck2/cmind/repo/automation/automation/cmind_automation/module.py @@ -0,0 +1,83 @@ +import os + +from cmind.module import Module +from cmind import utils + +class CModule(Module): + """ + OS automation actions + """ + + ############################################################ + def __init__(self, cmind, module_name): + super().__init__(cmind, module_name) + + ############################################################ + def add(self, i): + """ + Add automation + + Args: + (artifact) (str) - repository name + + """ + + import shutil + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + parsed_artifact = i.get('parsed_artifact',[]) + + artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') + + module_name = artifact_obj[0] if artifact_obj[0]!='' else artifact_obj[1] + + # Add placeholder + i['skip_con']=True + i['default']=True + + i['meta']={'automation_alias':self.meta['alias'], + 'automation_uid':self.meta['uid'], + 'module_name':self.cmind.cfg['cmind_python_module_prefix']+module_name, + 'tags':['automation',module_name]} + + r_obj=self.cmind.access(i) + if r_obj['return']>0: return r_obj + + new_automation_path = r_obj['path'] + + if con: + print ('Created automation in {}'.format(new_automation_path)) + + # Create Python module holder + module_holder_path = os.path.join(new_automation_path, + self.cmind.cfg['cmind_python_module_prefix']+module_name) + if not os.path.isdir(module_holder_path): + os.makedirs(module_holder_path) + + if con: + print ('Created module directory in {}'.format(module_holder_path)) + + # Copy support files + original_path = os.path.dirname(self.path) + + for f in ['requirements.txt', 'setup.py']: + f1 = os.path.join(original_path, f) + f2 = os.path.join(new_automation_path, f) + + if con: + print (' * Copying {} to {}'.format(f1, f2)) + + shutil.copyfile(f1,f2) + + # Copy module files + for f in ['__init__.py', 'module_dummy.py']: + f1 = os.path.join(original_path, self.cmind.cfg['cmind_python_module_prefix']+'automation', f) + f2 = os.path.join(new_automation_path, self.cmind.cfg['cmind_python_module_prefix']+module_name, f.replace('_dummy','')) + + if con: + print (' * Copying {} to {}'.format(f1, f2)) + + shutil.copyfile(f1,f2) + + return r_obj diff --git a/ck2/cmind/repo/automation/automation/cmind_automation/module_dummy.py b/ck2/cmind/repo/automation/automation/cmind_automation/module_dummy.py new file mode 100644 index 0000000000..e26cf3a93f --- /dev/null +++ b/ck2/cmind/repo/automation/automation/cmind_automation/module_dummy.py @@ -0,0 +1,27 @@ +import os + +from cmind.module import Module +from cmind import utils + +class CModule(Module): + """ + OS automation actions + """ + + ############################################################ + def __init__(self, cmind, module_name): + super().__init__(cmind, module_name) + + ############################################################ + def test(self, i): + """ + Test automation + + Args: + (artifact) (str) - repository name + + """ + + print (i) + + return {'return':0} diff --git a/ck2/cmind/repo/automation/automation/requirements.txt b/ck2/cmind/repo/automation/automation/requirements.txt new file mode 100644 index 0000000000..e6da3464e1 --- /dev/null +++ b/ck2/cmind/repo/automation/automation/requirements.txt @@ -0,0 +1 @@ +cmind>=0.0.1 diff --git a/ck2/cmind/repo/automation/automation/setup.py b/ck2/cmind/repo/automation/automation/setup.py new file mode 100644 index 0000000000..ac07b4200a --- /dev/null +++ b/ck2/cmind/repo/automation/automation/setup.py @@ -0,0 +1,57 @@ +import os +import sys +import re + +from setuptools import find_packages, setup, convert_path + +# Get version +current_path = os.path.abspath(os.path.dirname(__file__)) + +# Get all dependencies +requirements = [] + +with open(os.path.join(current_path, "requirements.txt"), "r", encoding="utf-8") as f: + for req in f: + if not req.startswith("--") and not req.startswith("#"): + requirements.append(req) + +# Attempt to get the name of the module +module_name = '' +dirs = os.listdir(current_path) +for d in dirs: + if os.path.isdir(d) and d.startswith('cmind_') and 'egg-info' not in d: + module_name=d + break + +############################################################ +# Add all directories in "automations" to the distribution + +setup( + name=module_name, + + author="OctoML", + author_email="grigori@octoml.ai", + + version='0.0.1', + + description="CM automation", + + license="Apache 2.0", + + url="TBA", + + python_requires="", # do not force for testing + + packages=find_packages(exclude=["tests*", "docs*"]), + + include_package_data=False, + + package_data={module_name: ['../_cm.json']}, + + install_requires=requirements, + + zip_safe=False, + + keywords="collective mind,cmind,automation,actions,meta descriptions,JSON,YAML,python", + +) diff --git a/ck2/cmind/repo/automation/ck/_cm.json b/ck2/cmind/repo/automation/ck/_cm.json new file mode 100644 index 0000000000..0e1ccd2c7f --- /dev/null +++ b/ck2/cmind/repo/automation/ck/_cm.json @@ -0,0 +1,12 @@ +{ + "alias": "ck", + "automation_alias": "automation", + "automation_uid": "bbeb15d8f0a944a4", + "module_name": "cmind_ck", + "tags": [ + "automation", + "ck" + ], + "uid": "1818c39eaf3a4a78", + "use_any_func":true +} \ No newline at end of file diff --git a/ck2/cmind/repo/automation/ck/cmind_ck/__init__.py b/ck2/cmind/repo/automation/ck/cmind_ck/__init__.py new file mode 100644 index 0000000000..e6a7f8fd52 --- /dev/null +++ b/ck2/cmind/repo/automation/ck/cmind_ck/__init__.py @@ -0,0 +1,7 @@ +from cmind import utils + +def init(cmind): + + from .module import CModule + + return utils.init_module(CModule, cmind, __file__) diff --git a/ck2/cmind/repo/automation/ck/cmind_ck/module.py b/ck2/cmind/repo/automation/ck/cmind_ck/module.py new file mode 100644 index 0000000000..c76f52f6b2 --- /dev/null +++ b/ck2/cmind/repo/automation/ck/cmind_ck/module.py @@ -0,0 +1,34 @@ +import os + +from cmind.module import Module +from cmind import utils + +class CModule(Module): + """ + OS automation actions + """ + + ############################################################ + def __init__(self, cmind, module_name): + super().__init__(cmind, module_name) + + ############################################################ + def any(self, i): + """ + Use as wrapper to CK + + Args: + (artifact) (str) - repository name + + """ + + import ck.kernel as ck + + artifact = i.get('artifact','') + + i['cid']=artifact + + if 'out' not in i: + i['out']='con' + + return ck.access(i) diff --git a/ck2/cmind/repo/automation/ck/requirements.txt b/ck2/cmind/repo/automation/ck/requirements.txt new file mode 100644 index 0000000000..28358fbe7a --- /dev/null +++ b/ck2/cmind/repo/automation/ck/requirements.txt @@ -0,0 +1,2 @@ +cmind>=0.0.1 +ck diff --git a/ck2/cmind/repo/automation/ck/setup.py b/ck2/cmind/repo/automation/ck/setup.py new file mode 100644 index 0000000000..34186a5d49 --- /dev/null +++ b/ck2/cmind/repo/automation/ck/setup.py @@ -0,0 +1,57 @@ +import os +import sys +import re + +from setuptools import find_packages, setup, convert_path + +# Get version +current_path = os.path.abspath(os.path.dirname(__file__)) + +# Get all dependencies +requirements = [] + +with open(os.path.join(current_path, "requirements.txt"), "r", encoding="utf-8") as f: + for req in f: + if not req.startswith("--") and not req.startswith("#"): + requirements.append(req) + +# Attempt to get the name of the module +module_name = '' +dirs = os.listdir(current_path) +for d in dirs: + if os.path.isdir(d) and d.startswith('cmind_') and 'egg-info' not in d: + module_name=d + break + +############################################################ +# Add all directories in "automations" to the distribution + +setup( + name=module_name, + + author="OctoML", + author_email="grigori@octoml.ai", + + version='0.0.1', + + description="CM CK wrapper", + + license="Apache 2.0", + + url="TBA", + + python_requires="", # do not force for testing + + packages=find_packages(exclude=["tests*", "docs*"]), + + include_package_data=False, + + package_data={module_name: ['../_cm.json']}, + + install_requires=requirements, + + zip_safe=False, + + keywords="collective mind,cmind,automation,actions,meta descriptions,JSON,YAML,python", + +) diff --git a/ck2/cmind/repo/automation/kernel/_cm.json b/ck2/cmind/repo/automation/kernel/_cm.json new file mode 100644 index 0000000000..dbd486dcfa --- /dev/null +++ b/ck2/cmind/repo/automation/kernel/_cm.json @@ -0,0 +1,11 @@ +{ + "uid":"60cb625a46b38610", + "alias":"kernel", + "automation_uid": "bbeb15d8f0a944a4", + "automation_alias": "automation", + "module_name":"cmind_kernel", + "tags": [ + "automation", + "repo" + ] +} diff --git a/ck2/cmind/repo/automation/kernel/cmind_kernel/__init__.py b/ck2/cmind/repo/automation/kernel/cmind_kernel/__init__.py new file mode 100644 index 0000000000..e6a7f8fd52 --- /dev/null +++ b/ck2/cmind/repo/automation/kernel/cmind_kernel/__init__.py @@ -0,0 +1,7 @@ +from cmind import utils + +def init(cmind): + + from .module import CModule + + return utils.init_module(CModule, cmind, __file__) diff --git a/ck2/cmind/repo/automation/kernel/cmind_kernel/module.py b/ck2/cmind/repo/automation/kernel/cmind_kernel/module.py new file mode 100644 index 0000000000..60f6f389a8 --- /dev/null +++ b/ck2/cmind/repo/automation/kernel/cmind_kernel/module.py @@ -0,0 +1,27 @@ +import os + +from cmind.module import Module +from cmind import utils + +class CModule(Module): + """ + Kernel automation actions + """ + + ############################################################ + def __init__(self, cmind, module_name): + super().__init__(cmind, module_name) + + ############################################################ + def uid(self, i): + """ + Generate CM UID + """ + + r = utils.gen_uid() + + if self.cmind.con: + print (r['uid']) + + return r + diff --git a/ck2/cmind/repo/automation/kernel/requirements.txt b/ck2/cmind/repo/automation/kernel/requirements.txt new file mode 100644 index 0000000000..e6da3464e1 --- /dev/null +++ b/ck2/cmind/repo/automation/kernel/requirements.txt @@ -0,0 +1 @@ +cmind>=0.0.1 diff --git a/ck2/cmind/repo/automation/kernel/setup.py b/ck2/cmind/repo/automation/kernel/setup.py new file mode 100644 index 0000000000..878bca7989 --- /dev/null +++ b/ck2/cmind/repo/automation/kernel/setup.py @@ -0,0 +1,57 @@ +import os +import sys +import re + +from setuptools import find_packages, setup, convert_path + +# Get version +current_path = os.path.abspath(os.path.dirname(__file__)) + +# Get all dependencies +requirements = [] + +with open(os.path.join(current_path, "requirements.txt"), "r", encoding="utf-8") as f: + for req in f: + if not req.startswith("--") and not req.startswith("#"): + requirements.append(req) + +# Attempt to get the name of the module +module_name = '' +dirs = os.listdir(current_path) +for d in dirs: + if os.path.isdir(d) and d.startswith('cmind_') and 'egg-info' not in d: + module_name=d + break + +############################################################ +# Add all directories in "automations" to the distribution + +setup( + name=module_name, + + author="OctoML", + author_email="grigori@octoml.ai", + + version='0.0.1', + + description="CM repo automation", + + license="Apache 2.0", + + url="TBA", + + python_requires="", # do not force for testing + + packages=find_packages(exclude=["tests*", "docs*"]), + + include_package_data=False, + + package_data={module_name: ['../_cm.json']}, + + install_requires=requirements, + + zip_safe=False, + + keywords="collective mind,cmind,automation,actions,meta descriptions,JSON,YAML,python", + +) diff --git a/ck2/cmind/repo/automation/repo/_cm.json b/ck2/cmind/repo/automation/repo/_cm.json new file mode 100644 index 0000000000..6c2c116161 --- /dev/null +++ b/ck2/cmind/repo/automation/repo/_cm.json @@ -0,0 +1,11 @@ +{ + "uid":"55c3e27e8a140e48", + "alias":"repo", + "automation_uid": "bbeb15d8f0a944a4", + "automation_alias": "automation", + "module_name":"cmind_repo", + "tags": [ + "automation", + "repo" + ] +} diff --git a/ck2/cmind/repo/automation/repo/cmind_repo/__init__.py b/ck2/cmind/repo/automation/repo/cmind_repo/__init__.py new file mode 100644 index 0000000000..e6a7f8fd52 --- /dev/null +++ b/ck2/cmind/repo/automation/repo/cmind_repo/__init__.py @@ -0,0 +1,7 @@ +from cmind import utils + +def init(cmind): + + from .module import CModule + + return utils.init_module(CModule, cmind, __file__) diff --git a/ck2/cmind/repo/automation/repo/cmind_repo/module.py b/ck2/cmind/repo/automation/repo/cmind_repo/module.py new file mode 100644 index 0000000000..b0c363530f --- /dev/null +++ b/ck2/cmind/repo/automation/repo/cmind_repo/module.py @@ -0,0 +1,495 @@ +import os + +from cmind.module import Module +from cmind import utils + +class CModule(Module): + """ + OS automation actions + """ + + ############################################################ + def __init__(self, cmind, module_name): + super().__init__(cmind, module_name) + + ############################################################ + def pull(self, i): + """ + Pull repo + + Args: + (artifact) (str) - repository name + (url) (str) - URL of a repository + (branch) (str) - Git branch + (checkout) (str) - Git checkout + (name) (str) - user-friendly name + (prefix) (str) - extra directory to keep CM artifacts + + """ + + alias = i.get('artifact','') + url = i.get('url','') + name = i.get('name','') + prefix = i.get('prefix','') + + if url == '': + if alias != '': + url = self.cmind.cfg['repo_url_prefix'] + + if '@' not in alias: + alias = self.cmind.cfg['repo_url_org'] + '@' + alias + + url += alias.replace('@','/') + else: + if alias == '': + # Get alias from URL + alias = url + if alias.endswith('.git'): alias=alias[:-4] + + j = alias.find('//') + if j>=0: + j1 = alias.find('/', j+2) + if j1>=0: + alias = alias[j1+1:].replace('/','@') + + if url == '': + return {'return':1, 'error':'TBD: no URL - need to update all Git repos'} + + # Branch and checkout + branch = i.get('branch','') + checkout = i.get('checkout','') + + if self.cmind.con: + print (self.cmind.cfg['line']) + print ('Alias: {}'.format(alias)) + print ('URL: {}'.format(url)) + print ('Branch: {}'.format(branch)) + print ('Checkout: {}'.format(checkout)) + print ('') + + # Prepare path to repo + repos = self.cmind.repos + + r = repos.pull(alias = alias, url = url, branch = branch, checkout = checkout, con = self.cmind.con, name=name, prefix=prefix) + if r['return']>0: return r + + repo_meta = r['meta'] + + return {'return':0, 'meta':repo_meta} + + ############################################################ + def search(self, i): + """ + List repos + + Args: + (verbose) (bool) - if True show extra info about repositories + + """ + + lst = [] + + parsed_artifact = i.get('parsed_artifact',[]) + + artifact_obj = parsed_artifact[0] if len(parsed_artifact)>0 else ('','') + artifact_repo = parsed_artifact[1] if len(parsed_artifact)>1 else None + + artifact_repo_wildcards = False + if artifact_repo is not None: + if '*' in artifact_repo[0] or '?' in artifact_repo[0]: + artifact_repo_wildcards = True + + for repo in self.cmind.repos.lst: + meta = repo.meta + + uid = meta['uid'] + alias = meta.get('alias','') + + r = utils.match_objects(uid = uid, + alias = alias, + uid2 = artifact_obj[1], + alias2 = artifact_obj[0]) + if r['return']>0: return r + + if r['match']: + lst.append(repo) + + # Output paths to console if CLI or forced + if self.cmind.con and not i.get('skip_con', False): + for l in lst: + meta = l.meta + alias = meta.get('alias','') + + if i.get('verbose',False): + + uid = meta['uid'] + name = meta.get('name','') + path = repo.path + git = meta.get('git',False) + + print ('{},{} "{}" {}'.format(alias,uid,name,path)) + else: + print ('{} = {}'.format(alias, l.path)) + + return {'return':0, 'lst':lst} + + ############################################################ + def update(self, i): + """ + Update repos + + Args: + (verbose) (bool) - if True show extra info about repositories + + """ + + for repo in self.cmind.repos.lst: + meta = repo.meta + + alias = meta.get('alias','') + + git = meta.get('git', False) + + if git: + if self.cmind.con: + print ('Updating "{}" ...'.format(alias)) + print ('') + r = self.cmind.repos.pull(alias = alias, con = self.cmind.con) + if r['return']>0: return r + + return {'return':0, 'lst':self.cmind.repos.lst} + + ############################################################ + def delete(self, i): + """ + Delete repo (just unlink or remove content too) + + Args: + (artifact) (str) - repository name + (all) (bool) - if True, remove with content otherwise just unlink + + """ + + alias = i.get('artifact', '') + remove_all = i.get('all', '') + + # Prepare path to repo + repos = self.cmind.repos + + r = repos.delete(alias = alias, remove_all = remove_all, con = self.cmind.con) + + return r + + ############################################################ + def init(self, i): + """ + Initialize repo + + Args: + (artifact) (str) - repository name + (path) (str) - if !='' use this non-standard path + (name) (str) - user-friendly name + (prefix) (str) - extra directory to keep CM artifacts + """ + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + alias = i.get('artifact', '') + path = i.get('path', '') + name = i.get('name','') + prefix = i.get('prefix','') + + uid = i.get('uid','') + if uid =='': + r=utils.gen_uid() + if r['return']>0: return r + uid = r['uid'] + + if alias == '': alias = uid + + if con: + print (self.cmind.cfg['line']) + print ('Alias: {}'.format(alias)) + print ('UID: {}'.format(uid)) + print ('Name: {}'.format(name)) + print ('Prefix: {}'.format(prefix)) + print ('') + + # Prepare path to repo + repos = self.cmind.repos + + r = repos.init(alias = alias, uid = uid, path = path, con = self.cmind.con, name=name, prefix=prefix) + return r + + ############################################################ + def add(self, i): + """ + Initialize repo + + Args: + (artifact) (str) - repository name + (path) (str) - if !='' use this non-standard path + (name) (str) - user-friendly name + (prefix) (str) - extra directory to keep CM artifacts + """ + + return self.init(i) + + ############################################################ + def pack(self, i): + """ + Pack repo for further distribution + + Args: + (artifact) (str) - repository name + """ + + import zipfile + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + # Search repository + i['skip_con']=True + r = self.search(i) + if r['return']>0: return r + + lst = r['lst'] + + if len(lst)==0: + return {'return':16, 'error':'no repsitories found'} + elif len(lst)>1: + return {'return':16, 'error':'more than 1 repository found'} + + repo = lst[0] + + repo_path = repo.path + + pack_file = self.cmind.cfg['default_repo_pack'] + + if con: + print ('Packing repo from {} to {} ...'.format(repo_path, pack_file)) + + r = utils.list_all_files({'path': repo_path, 'all': 'yes'}) + if r['return'] > 0: return r + + files = r['list'] + + # Prepare zip archive + try: + f = open(pack_file, 'wb') + z = zipfile.ZipFile(f, 'w', zipfile.ZIP_DEFLATED) + + for fn in files: + path_to_file = os.path.join(repo_path, fn) + z.write(path_to_file, fn, zipfile.ZIP_DEFLATED) + + z.close() + f.close() + + except Exception as e: + return {'return': 1, 'error': 'failed to pack repo: {}'.format(format(e))} + + return {'return':0} + + ############################################################ + def unpack(self, i): + """ + Unpack repository + + Args: + (artifact) (str) - CM repo pack file name + """ + + import zipfile + + con = True if self.cmind.con and not i.get('skip_con', False) else False + + pack_file = self.cmind.cfg['default_repo_pack'] + + if not os.path.isfile(pack_file): + return {'return':16, 'error':'CM repo pack file not found {}'.format(pack_file)} + + # Attempt to read cmr.json + repo_pack_file = open(pack_file, 'rb') + repo_pack_zip = zipfile.ZipFile(repo_pack_file) + + repo_pack_desc = self.cmind.cfg['file_meta_repo'] + + files=repo_pack_zip.namelist() + + repo_meta = {} + + file_yaml = repo_pack_desc + '.yaml' + if file_yaml in files: + repo_meta_yaml = repo_pack_zip.read(file_yaml) + + import yaml + + repo_meta.update(yaml.load(repo_meta_yaml, Loader=yaml.FullLoader)) + + file_json = repo_pack_desc + '.json' + if file_json in files: + repo_meta_json = repo_pack_zip.read(file_json) + + import json + + repo_meta.update(json.load(repo_meta_json)) + + # Get meta info + uid = repo_meta['uid'] + alias = repo_meta['alias'] + name = repo_meta.get('name','') + prefix = repo_meta.get('prefix','') + + # Check if repo exists + r = self.search({'parsed_artifact':[(alias,uid)], 'skip_con':True}) + if r['return']>0: return r + + lst = r['lst'] + + if len(lst)>0: + return {'return':1, 'error':'Repository already exists'} + + # Initialize repository + r_new = self.init({'artifact':alias, + 'uid':uid, + 'prefix':prefix, + 'skip_con':True}) + if r_new['return']>0: return r_new + + repo_path = r_new['path_to_repo'] + + if con: + print ('Unpacking {} to {} ...'.format(pack_file, repo_path)) + + # Unpacking zip + for f in files: + if not f.startswith('.') and not f.startswith('/') and not f.startswith('\\'): + file_path = os.path.join(repo_path, f) + if f.endswith('/'): + # create directory + if not os.path.exists(file_path): + os.makedirs(file_path) + else: + dir_name = os.path.dirname(file_path) + if not os.path.exists(dir_name): + os.makedirs(dir_name) + + # extract file + file_out = open(file_path, 'wb') + file_out.write(repo_pack_zip.read(f)) + file_out.close() + + repo_pack_zip.close() + repo_pack_file.close() + + return r_new + + ############################################################################## + def convert_ck_to_cm(self, i): + """ + Convert CK repos to CM repos + + Input: { + } + + Output: { + return - return code = 0, if successful + > 0, if error + (error) - error text if return > 0 + } + + """ + + import ck.kernel as ck + import cmind + import os + + r=ck.access({'action':'search', + 'module_uoa':'repo', + 'add_meta':'yes'}) + if r['return']>0: return r + + lst=r['lst'] + + for l in lst: + ruoa=l['data_uoa'] + ruid=l['data_uid'] + + rmeta=l['meta'] + + rpath=rmeta.get('path','') + + if rpath!='' and os.path.isdir(rpath): + print ('********************************************************') + print (ruoa) + print (rpath) + + r=cmind.access({'automation':'repo', + 'action':'init', + 'artifact':ruoa, + 'path':rpath}) + if r['return']>0: return r + + dir1 = os.listdir(rpath) + + for d1 in dir1: + if d1!='.cm': + dd1=os.path.join(rpath, d1) + if os.path.isdir(dd1): + + dir2 = os.listdir(dd1) + + for d2 in dir2: + if d2!='.cm': + dd2=os.path.join(dd1, d2, '.cm') + if os.path.isdir(dd2): + dmeta = {} + + broken = False + for f in ['info.json','meta.json']: + dd2a=os.path.join(dd2, f) + + if os.path.isfile(dd2a): + + r=ck.load_json_file({'json_file':dd2a}) + if r['return']>0: return r + + dmeta.update(r['dict']) + else: + broken=True + break + + if broken or \ + 'backup_data_uid' not in dmeta or \ + 'backup_module_uid' not in dmeta or \ + 'backup_module_uoa' not in dmeta: + + print ('Ignored: '+dd2) + else: + backup_data_uid = dmeta['backup_data_uid'] + + backup_module_uid = dmeta['backup_module_uid'] + backup_module_uoa = dmeta['backup_module_uoa'] + + dmeta2 = {} + for k in dmeta: + if not k.startswith('backup_') and not k.startswith('cm_'): + dmeta2[k]=dmeta[k] + + dmeta2['uid']=backup_data_uid + + if not ck.is_uid(d2): + dmeta2['alias']=d2 + + dmeta2['automation_alias']=backup_module_uoa + dmeta2['automation_uid']=backup_module_uid + + cm_meta_file = os.path.join(dd1, d2, '_cm.json') + + print (cm_meta_file) + + r=ck.save_json_to_file({'json_file':cm_meta_file, 'dict': dmeta2, 'sort_keys':'yes'}) + if r['return']>0: return r + + return {'return':0} diff --git a/ck2/cmind/repo/automation/repo/requirements.txt b/ck2/cmind/repo/automation/repo/requirements.txt new file mode 100644 index 0000000000..e6da3464e1 --- /dev/null +++ b/ck2/cmind/repo/automation/repo/requirements.txt @@ -0,0 +1 @@ +cmind>=0.0.1 diff --git a/ck2/cmind/repo/automation/repo/setup.py b/ck2/cmind/repo/automation/repo/setup.py new file mode 100644 index 0000000000..878bca7989 --- /dev/null +++ b/ck2/cmind/repo/automation/repo/setup.py @@ -0,0 +1,57 @@ +import os +import sys +import re + +from setuptools import find_packages, setup, convert_path + +# Get version +current_path = os.path.abspath(os.path.dirname(__file__)) + +# Get all dependencies +requirements = [] + +with open(os.path.join(current_path, "requirements.txt"), "r", encoding="utf-8") as f: + for req in f: + if not req.startswith("--") and not req.startswith("#"): + requirements.append(req) + +# Attempt to get the name of the module +module_name = '' +dirs = os.listdir(current_path) +for d in dirs: + if os.path.isdir(d) and d.startswith('cmind_') and 'egg-info' not in d: + module_name=d + break + +############################################################ +# Add all directories in "automations" to the distribution + +setup( + name=module_name, + + author="OctoML", + author_email="grigori@octoml.ai", + + version='0.0.1', + + description="CM repo automation", + + license="Apache 2.0", + + url="TBA", + + python_requires="", # do not force for testing + + packages=find_packages(exclude=["tests*", "docs*"]), + + include_package_data=False, + + package_data={module_name: ['../_cm.json']}, + + install_requires=requirements, + + zip_safe=False, + + keywords="collective mind,cmind,automation,actions,meta descriptions,JSON,YAML,python", + +) diff --git a/ck2/cmind/repo/cmr.yaml b/ck2/cmind/repo/cmr.yaml new file mode 100644 index 0000000000..bc74e21a1c --- /dev/null +++ b/ck2/cmind/repo/cmr.yaml @@ -0,0 +1,3 @@ +uid: "36b263b05174aef9" +alias: default +name: "default CM repository" diff --git a/ck2/cmind/repos.py b/ck2/cmind/repos.py new file mode 100644 index 0000000000..c1f38f908d --- /dev/null +++ b/ck2/cmind/repos.py @@ -0,0 +1,309 @@ +# Collective Mind repositories + +import os + +from cmind.repo import Repo +from cmind import utils + +class Repos: + ############################################################ + def __init__(self, path, cfg, path_to_default_repo = ''): + """ + Initialize class + """ + + self.path = path + self.cfg = cfg + + self.full_path_to_repos = '' + + # Paths to CM repos + self.paths = [] + + # List of initialized repositories + self.lst = [] + + # Potential path to default repo + self.path_to_default_repo = path_to_default_repo + + self.full_path_to_repo_paths = '' + + ############################################################ + def load(self, init = False): + """ + Load or initialize file with repositories + + """ + + # If reload after updating repos + if init: + self.paths = [] + self.lst = [] + + # Check if home directory exists. Create it otherwise. + if not os.path.isdir(self.path): + os.makedirs(self.path) + + # Check repos holder + full_path_to_repos = os.path.join(self.path, + self.cfg['dir_repos']) + + self.full_path_to_repos = full_path_to_repos + + # If placeholder for repos doesn't exist, create it + if not os.path.isdir(full_path_to_repos): + os.makedirs(full_path_to_repos) + + # Search if there is a file with repos + full_path_to_repo_paths = os.path.join(self.path, self.cfg['file_repos']) + self.full_path_to_repo_paths = full_path_to_repo_paths + + if os.path.isfile(full_path_to_repo_paths): + r = utils.load_json(full_path_to_repo_paths) + if r['return']>0: return r + + self.paths = r['meta'] + else: + r = utils.save_json(full_path_to_repo_paths, meta = self.paths) + if r['return']>0: return r + + # Prepare path for local repo + path_local_repo = os.path.join(full_path_to_repos, self.cfg['local_repo_name']) + + if not os.path.isdir(path_local_repo): + os.makedirs(path_local_repo) + + path_local_repo_meta = os.path.join(path_local_repo, self.cfg['file_meta_repo']+'.yaml') + + if not os.path.isfile(path_local_repo_meta): + r = utils.save_yaml(path_local_repo_meta, + meta = self.cfg['local_repo_meta']) + if r['return']>0: return r + + if path_local_repo not in self.paths: + self.paths.insert(0, path_local_repo) + + # Check default repo + if self.path_to_default_repo != '' and os.path.isdir(self.path_to_default_repo): + self.paths.insert(0, self.path_to_default_repo) + + # Check that repository exists and load meta description + for path_to_repo in self.paths: + # First try concatenated path and then full path (if imported) + found = False + for full_path_to_repo in [os.path.join(full_path_to_repos, path_to_repo), + path_to_repo]: + if os.path.isdir(full_path_to_repo): + # Load description + repo = Repo(full_path_to_repo, self.cfg) + + r = repo.load() + if r['return'] >0 : return r + + # Set only after all initializations + self.lst.append(repo) + + found = True + break + + # Repo path exists but repo itself doesn't exist - fail + if not found: + return {'return':1, 'error': 'repository path {} not found (check file {})'.format(path_to_repo, full_path_to_repo_paths)} + + return {'return':0} + + ############################################################ + def process(self, repo_path, mode='add'): + """ + Process file with repo list + + """ + + # Load clean file with repo paths + r = utils.load_json(self.full_path_to_repo_paths) + if r['return']>0: return r + + paths = r['meta'] + + modified = False + + if mode == 'add': + if repo_path not in paths: + paths.append(repo_path) + modified = True + elif mode == 'delete': + new_paths = [] + for p in paths: + if p==repo_path: + modified = True + else: + new_paths.append(p) + paths=new_paths + + if modified: + r = utils.save_json(self.full_path_to_repo_paths, meta = paths) + if r['return']>0: return r + + # Reload repos + self.load(init=True) + + return {'return':0} + + ############################################################ + def pull(self, alias, url = '', branch = '', checkout = '', con = False, name = '', prefix = ''): + """ + Pull or clone repository + + """ + + # Prepare path + path_to_repo = os.path.join(self.full_path_to_repos, alias) + + if con: + print ('Local path: '+path_to_repo) + print ('') + + cur_dir = os.getcwd() + + clone=False + if os.path.isdir(path_to_repo): + # Attempt to update + os.chdir(path_to_repo) + + cmd = 'git pull' + else: + # Attempt to clone + clone = True + + os.chdir(self.full_path_to_repos) + + cmd = 'git clone '+url+' '+alias + + if con: + print (cmd) + print ('') + + os.system(cmd) + + if con: + print ('') + + # Check if repo description exists + path_to_repo_desc = os.path.join(path_to_repo, self.cfg['file_meta_repo']) + + r=utils.is_file_json_or_yaml(file_name = path_to_repo_desc) + if r['return']>0: return r + + if not r['is_file']: + # Prepare meta + r=utils.gen_uid() + if r['return']>0: return r + + repo_uid = r['uid'] + + meta = { + 'uid': repo_uid, + 'alias': alias, + 'git':True, + } + + if name!='': meta['name']=name + if prefix!='': meta['prefix']=prefix + + r=utils.save_yaml(path_to_repo_desc + '.yaml', meta=meta) + if r['return']>0: return r + else: + # Load meta from the repository + r=utils.load_yaml_and_json(file_name_without_ext=path_to_repo_desc) + if r['return']>0: return r + + meta = r['meta'] + + # Update repo list + # TBD: make it more safe (reload and save) + r = self.process(path_to_repo, 'add') + if r['return']>0: return r + + # Go back to original directory + os.chdir(cur_dir) + + return {'return':0, 'meta':meta} + + ############################################################ + def init(self, alias, uid, path = '', con = False, name = '', prefix = ''): + """ + Init or clone repository + + """ + + # Prepare path + if uid == '': + r=utils.gen_uid() + if r['return']>0: return r + uid = r['uid'] + + repo_name=alias if alias!='' else uid + + path_to_repo = os.path.join(self.full_path_to_repos, repo_name) if path=='' else path + path_to_repo_desc = os.path.join(path_to_repo, self.cfg['file_meta_repo']) + + if con: + print ('Local path: '+path_to_repo) + print ('') + + if not os.path.isdir(path_to_repo): + os.makedirs(path_to_repo) + + meta = { + 'uid': uid, + 'alias': alias, + 'git':False, + } + + if name!='': meta['name']=name + if prefix!='': meta['prefix']=prefix + + r=utils.save_yaml(path_to_repo_desc + '.yaml', meta=meta) + if r['return']>0: return r + + # Update repo list + # TBD: make it more safe (reload and save) + r = self.process(path_to_repo, 'add') + if r['return']>0: return r + + return {'return':0, 'meta':meta, 'path_to_repo': path_to_repo, 'path_to_repo_desc': path_to_repo_desc} + + ############################################################ + def delete(self, alias, remove_all = False, con = False): + """ + Delete repository with or without content + + """ + + # Prepare path + path_to_repo = os.path.join(self.full_path_to_repos, alias) + + if con: + print ('Local path: '+path_to_repo) + print ('') + + if path_to_repo not in self.paths: + return {'return':16, 'error':'repository not found'} + + # TBD: make it more safe (reload and save) + r = self.process(path_to_repo, 'delete') + if r['return']>0: return r + + # Check if remove all + if remove_all: + import shutil + + if con: + print ('Deleting repository content ...') + + shutil.rmtree(path_to_repo) + else: + if con: + print ('Repository was unlinked from CM but the content was not deleted.') + + return {'return':0} diff --git a/ck2/cmind/utils.py b/ck2/cmind/utils.py new file mode 100644 index 0000000000..9667700be2 --- /dev/null +++ b/ck2/cmind/utils.py @@ -0,0 +1,619 @@ +# Some functionality may be reused from the CK framework + +import os + +ERROR_UNKNOWN_FILE_EXTENSION = 1 +ERROR_PATH_NOT_FOUND = 2 +ERROR_FILE_NOT_FOUND = 16 + +########################################################################### +def load_yaml_and_json(file_name_without_ext, check_if_exists = False, encoding = 'utf8'): + + meta = {} + + for file_ext in [('.yaml', load_yaml), + ('.json', load_json)]: + file_name = file_name_without_ext + file_ext[0] + + r = file_ext[1](file_name, check_if_exists = True, encoding = encoding) # To avoid failing if doesn't exist + if r['return'] > 0 and r['return'] != ERROR_FILE_NOT_FOUND: return r + + meta.update(r.get('meta', {})) + + return {'return':0, 'meta':meta} + +########################################################################### +def is_file_json_or_yaml(file_name): + + for file_ext in ['.yaml', '.json']: + file_path = file_name + file_ext + + if os.path.isfile(file_path): + return {'return':0, 'is_file':True, 'path':file_path} + + return {'return':0, 'is_file':False} + +########################################################################### +def load_json_or_yaml(file_name, check_if_exists = False, encoding = 'utf8'): + + if file_name.endswith('.json'): + return load_json(file_name, check_if_exists = check_if_exists, encoding = encoding) + elif file_name.endswith('.yaml'): + return load_yaml(file_name, check_if_exists = check_if_exists, encoding = encoding) + + return {'return':ERROR_UNKNOWN_FILE_EXTENSION, 'error':'file extension must be .json or .yaml in {}'.format(file_name)} + +########################################################################### +def save_json_or_yaml(file_name, meta, sort_keys=False, encoding = 'utf8'): + if file_name.endswith('.json'): + return save_json(file_name, meta, sort_keys, encoding = encoding) + elif file_name.endswith('.yaml'): + return save_yaml(file_name, meta, sort_keys, encoding = encoding) + + return {'return':ERROR_UNKNOWN_FILE_EXTENSION, 'error':'unknown file extension'} + +########################################################################### +def load_json(file_name, check_if_exists = False, encoding='utf8'): + + if check_if_exists: + import os + if not os.path.isfile(file_name): + return {'return':ERROR_FILE_NOT_FOUND, 'error':'File {} not found'.format(file_name)} + + import json + + with open(file_name, encoding=encoding) as jf: + meta = json.load(jf) + + return {'return':0, + 'meta': meta} + +########################################################################### +def save_json(file_name, meta={}, indent=2, sort_keys=True, encoding = 'utf8'): + + import json + + with open(file_name, 'w', encoding = encoding) as jf: + jf.write(json.dumps(meta, indent=indent, sort_keys=sort_keys)) + + return {'return':0} + +########################################################################### +def load_yaml(file_name, check_if_exists = False, encoding = 'utf8'): + + if check_if_exists: + import os + if not os.path.isfile(file_name): + return {'return':ERROR_FILE_NOT_FOUND, 'error':'File {} not found'.format(file_name)} + + import yaml + + with open(file_name, 'rt', encoding = encoding) as yf: + meta = yaml.load(yf, Loader=yaml.FullLoader) + + return {'return':0, + 'meta': meta} + +########################################################################### +def save_yaml(file_name, meta={}, sort_keys=True, encoding = 'utf8'): + + import yaml + + with open(file_name, 'w', encoding = encoding) as yf: + meta = yaml.dump(meta, yf) + + return {'return':0} + +########################################################################### +def check_and_create_dir(path): + """ + Create directories if path doesn't exist + """ + + if not os.path.isdir(path): + os.makedirs(path) + + return {'return':0} + +########################################################################### +def find_file_in_dir_and_above(filename, + path=""): + """ + Find file in the current directory or above + + Args: + filename (str) + path (str) + + Returns: + (dict) return (int): 0 - if found + 16 - if not found + (error) (str) + + path (str): path where file is found + + path_to_file (str): path to file + """ + + if path == "": + path = os.getcwd() + + if not os.path.isdir(path): + return {'return':ERROR_PATH_NOT_FOUND, 'error': 'path not found'} + + path = os.path.realpath(path) + + while True: + test_path = os.path.join(path, filename) + + if os.path.isfile(test_path): + return {'return':0, 'path': path, 'path_to_file': test_path} + + new_path, skip = os.path.split(path) + + if new_path == path: + break + + path = new_path + + return {'return':ERROR_FILE_NOT_FOUND, 'error': 'path not found'} + +############################################################################## +def list_all_files(i): + """List all files recursively in a given directory + (from CK framework) + + Args: + path (str): top level path + (file_name) (str): search for a specific file name + (pattern) (str): return only files with this pattern + (path_ext) (str): path extension (needed for recursion) + (limit) (str): limit number of files (if directories with a large number of files) + (number) (int): current number of files + (all) (str): if 'yes' do not ignore special directories (like .cm) + (ignore_names) (list): list of names to ignore + (ignore_symb_dirs) (str): if 'yes', ignore symbolically linked dirs + (to avoid recursion such as in LLVM) + (add_path) (str) - if 'yes', add full path to the final list of files + + Returns: + (dict): Unified CK dictionary: + + return (int): return code = 0, if successful + > 0, if error + (error) (str): error text if return > 0 + + list (dict): dictionary of all files: + {"file_with_full_path":{"size":.., "path":..} + + sizes (dict): sizes of all files (the same order as above "list") + + number (int): (internal) total number of files in a current directory (needed for recursion) + + """ + + import sys + + number = 0 + if i.get('number', '') != '': + number = int(i['number']) + + inames = i.get('ignore_names', []) + + fname = i.get('file_name', '') + + limit = -1 + if i.get('limit', '') != '': + limit = int(i['limit']) + + a = {} + + iall = i.get('all', '') + + pe = '' + if i.get('path_ext', '') != '': + pe = i['path_ext'] + + po = i.get('path', '') + if sys.version_info[0] < 3: + po = unicode(po) + + pattern = i.get('pattern', '') + if pattern != '': + import fnmatch + + xisd = i.get('ignore_symb_dirs', '') + isd = False + if xisd == 'yes': + isd = True + + ap = i.get('add_path', '') + + try: + dirList = os.listdir(po) + except Exception as e: + None + else: + for fn in dirList: + p = os.path.join(po, fn) + if iall == 'yes' or fn not in cfg['special_directories']: + if len(inames) == 0 or fn not in inames: + if os.path.isdir(p): + if not isd or os.path.realpath(p) == p: + r = list_all_files({'path': p, 'all': iall, 'path_ext': os.path.join(pe, fn), + 'number': str(number), 'ignore_names': inames, 'pattern': pattern, + 'file_name': fname, 'ignore_symb_dirs': xisd, 'add_path': ap, 'limit': limit}) + if r['return'] > 0: + return r + a.update(r['list']) + else: + add = True + + if fname != '' and fname != fn: + add = False + + if pattern != '' and not fnmatch.fnmatch(fn, pattern): + add = False + + if add: + pg = os.path.join(pe, fn) + if os.path.isfile(p): + a[pg] = {'size': os.stat(p).st_size} + + if ap == 'yes': + a[pg]['path'] = po + + number = len(a) + if limit != -1 and number >= limit: + break + + return {'return': 0, 'list': a, 'number': str(number)} + +########################################################################### +def gen_uid(): + """ + Generate CM UID + """ + + import uuid + + return {'return':0, + 'uid':uuid.uuid4().hex[:16]} + +########################################################################### +def is_cm_uid(obj): + """ + Check if a string is a valid CM UID + + Args: + obj (str): CM alias or UID + + Returns: + (bool): True if a string is a valid CK UID + """ + + import re + + if len(obj) != 16: + return False + + pattern = r'[^\.a-f0-9]' + if re.search(pattern, obj.lower()): + return False + + return True + +########################################################################### +def parse_cm_object(obj, max_length = 2): + """ + Parse CM object + + Args: + obj (str): CM object + + CM sub-object = UID | alias | alias,UID | UID,alias + repo CM sub-object | CM sub-object + + Examples: + + cm os + cm 281d5c3e3f69d8e7 + cm os,281d5c3e3f69d8e7 + cm 281d5c3e3f69d8e7,os + + cm octoml@mlops,os + cm octoml@mlops,dbfa91645e429380:os,281d5c3e3f69d8e7 + cm dbfa91645e429380:281d5c3e3f69d8e7 + + Returns: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + cm_object (list): first argument: CM alias | UID + (second element: + + """ + + str_err='CM object {} is not recognized' + + cm_object = [] + + split_obj = obj.split(':') + + if len(split_obj) > max_length: + return {'return':1, 'error':str_err.format(obj)} + + for obj in split_obj: + sub_objects = obj.split(',') + + if len(sub_objects)==0 or len(sub_objects) > 2: + return {'return':1, 'error':str_err.format(obj)} + + if len(sub_objects)==1: + if is_cm_uid(sub_objects[0]): + sub_object = ('',sub_objects[0]) + else: + sub_object = (sub_objects[0],'') + elif len(sub_objects)==2: + if is_cm_uid(sub_objects[1]) or not is_cm_uid(sub_objects[0]): + sub_object = (sub_objects[0], sub_objects[1]) + elif is_cm_uid(sub_objects[0]) or not is_cm_uid(sub_objects[1]): + sub_object = (sub_objects[1], sub_objects[0]) + else: + return {'return':1, 'error':str_err.format(sub_objects)} + else: + return {'return':1, 'error':str_err.format(sub_objects)} + + cm_object.insert(0, sub_object) + + return {'return':0, 'cm_object':cm_object} + + +########################################################################### +def match_objects(uid, alias, uid2, alias2): + """ + Check if 2 CM objects match + + Args: + + alias can't have wildcards (real CM object) + alias2 can have wildcards (search) + + 281d5c3e3f69d8e7,* == 281d5c3e3f69d8e7,* + 281d5c3e3f69d8e7,os == ,os + ,os == 281d5c3e3f69d8e7,os + ,* != 281d5c3e3f69d8e7,* + + os + 281d5c3e3f69d8e7 + os,281d5c3e3f69d8e7 + 281d5c3e3f69d8e7,os + + + Returns: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + match (bool): True if 2 CM objects match (uid and/or alias) + + """ + + match = False + + if uid is None: uid = '' + if alias is None: alias = '' + if uid2 is None: uid2 = '' + if alias2 is None: alias2 = '' + + # We match first by UID no matter what the alias is (the last one can change) + if uid!='' and uid2!='': + if uid==uid2: + match = True + else: + # As soon as one UID is not there, we try to match by alias with wildcards + # Both aliases must be present otherwise ambiguity - we report is as no match + object2_has_wildcards = False + if '*' in alias2 or '?' in alias2: + object2_has_wildcards = True + + if object2_has_wildcards: + import fnmatch + + if fnmatch.fnmatch(alias, alias2): + match = True + else: + if alias2=='' or alias.lower()==alias2.lower(): + match = True + + return {'return':0, 'match': match} + +########################################################################### +def get_list_from_cli(i, key): + """ + Get list from a CLI + + Args: + + Returns: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + match (bool): True if 2 CM objects match (uid and/or alias) + + """ + + tags = i.get(key, []) + + if type(tags)!=list: + xtags = tags.split(',') + + tags = [t.strip() for t in xtags] + + return tags + +########################################################################### +def init_module(CModule, cmind, module_name): + """ + Initialize module + + Args: + + Returns: + return (int): return code == 0 if no error + >0 if error + + (error) (str): error string if return>0 + + """ + + + module = CModule(cmind, module_name) + + module_path = module.path + + # Try to load meta description + path_module_meta = os.path.join(os.path.dirname(module_path), cmind.cfg['file_cmeta']) + + r = is_file_json_or_yaml(file_name = path_module_meta) + if r['return']>0: return r + + module.meta = {} + + if r['is_file']: + # Load artifact class + r=load_yaml_and_json(path_module_meta) + if r['return']>0: return r + + module.meta = r['meta'] + + return {'return':0, 'module':module} + +############################################################################## +def merge_dicts(i): + """Merge intelligently dict1 with dict2 key by key in contrast with dict1.update(dict2) + Target audience: end users + + It can merge sub-dictionaries and lists instead of substituting them + + Args: + dict1 (dict): merge this dict with dict2 (will be directly modified!) + dict2 (dict): dict to be merged + append_lists (str): if 'yes', append lists instead of creating the new ones + ignore_keys (list): ignore keys + + Returns: + (dict): Unified CK dictionary: + + return (int): return code = 0, if successful + > 0, if error + (error) (str): error text if return > 0 + + dict1 (dict): dict1 passed through the function + + """ + + a = i['dict1'] + b = i['dict2'] + + append_lists=i.get('append_lists','') + + ignore_keys = i.get('ignore_keys',[]) + + for k in b: + if k in ignore_keys: + continue + v = b[k] + if type(v) is dict: + if k not in a: + a.update({k: b[k]}) + elif type(a[k]) == dict: + merge_dicts({'dict1': a[k], 'dict2': b[k], 'append_lists':append_lists}) + else: + a[k] = b[k] + elif type(v) is list: + if append_lists!='yes' or k not in a: + a[k] = [] + for y in v: + a[k].append(y) + else: + a[k] = b[k] + + return {'return': 0, 'dict1': a} + +########################################################################### +def process_meta_for_inheritance(i): + """Process meta for inheritance + + Args: + artifact (obj): CM artifact + meta (dict): original meta + cmind (obj): initialized CM to search for base artifacts + (base_recursion) (int): track recursion during inheritance + + Returns: + (dict): Unified CK dictionary: + + return (int): return code = 0, if successful + > 0, if error + (error) (str): error text if return > 0 + + dict (dict): CK updated meta with inheritance from base entries + (dict_orig) (dict): original CK meta if CK was updated with a base entry + + """ + + automation = i['automation'] + current_meta = i.get('meta',{}) + cmind = i['cmind'] + + base_entry = current_meta.get('_base_artifact','').strip() + + if base_entry!='': + base_recursion = int(i.get('base_recursion','0')) + + if base_recursion > 10: + return {'return':8, 'error':'inheritance recursion is too deep > 10 ({})'.format(i)} + + j=base_entry.find('::') + if j>0: + automation = base_entry[:j] + artifact = base_entry[j+2:] + else: + artifact = base_entry + + r = cmind.access({'automation':automation, + 'action':'search', + 'artifact':artifact, + 'ignore_inheritance':True, + 'skip_con':True}) + if r['return']>0: return r + + lst = r['list'] + + if len(lst)==0: + return {'return':1, 'error':'base artifact {} not found in {}'.format(artifact, current_meta['alias']+','+current_meta['uid'])} + + if len(lst)>1: + return {'return':1, 'error':'more than 1 base artifact {} found in {}'.format(artifact, current_meta['alias']+','+current_meta['uid'])} + + base_artifact = lst[0] + + # Load with meta and recursive inheritance + r = base_artifact.load(base_recursion = base_recursion + 1) + if r['return']>0: return r + + base_meta = base_artifact.meta + + r = merge_dicts({'dict1': base_meta, + 'dict2': current_meta}) + if r['return']>0: return r + + current_meta = base_meta + + return {'return':0, 'meta':current_meta} diff --git a/ck2/setup.py b/ck2/setup.py new file mode 100644 index 0000000000..2126772d2d --- /dev/null +++ b/ck2/setup.py @@ -0,0 +1,80 @@ +import os +import sys +import re + +from setuptools import find_packages, setup, convert_path + +# Get version +current_path = os.path.abspath(os.path.dirname(__file__)) + +with open(os.path.join(current_path, "cmind", "__init__.py"), encoding="utf-8") as f: + output = re.search(r'__version__ = ["\']([^"\']+)', f.read()) + + if not output: + raise ValueError("Error: can't find version in cmind/__init__.py") + + version = output.group(1) + +## Get all dependencies +#requirements = [] +# +#with open(os.path.join(current_path, "requirements.txt"), "r", encoding="utf-8") as f: +# for req in f: +# if not req.startswith("--") and not req.startswith("#"): +# requirements.append(req) + +############################################################ +# Add all directories in "automations" to the distribution + +root = 'cmind' + +repo=os.walk(os.path.join(root,'repo')) + +repo_dirs=[''] + +for artifact in repo: + directory=os.path.join(artifact[0], '*') + ignore=False + for ignore_dir in ['__pycache__', 'build', 'egg-info', 'dist']: + if ignore_dir in directory: + ignore=True + break + if not ignore: + repo_dirs.append(directory[len(root)+1:]) + +setup( + name="cmind", + + author="Grigori Fursin", + author_email="grigori@octoml.ai", + + version=version, + + description="cmind", + + license="Apache 2.0", + + long_description=open(convert_path('./README.md'), encoding="utf-8").read(), + long_description_content_type="text/markdown", + + url="https://github.com/mlcommons/ck/tree/master/ck2", + + python_requires="", # do not force for testing + + packages=['cmind'], + + include_package_data=False, + + package_data={'cmind': repo_dirs}, + + install_requires=['pyyaml'], + + entry_points={"console_scripts": [ + "cmind = cmind.cli:run", + "cm = cmind.cli:run" + ]}, + + zip_safe=False, + + keywords="collective mind,cmind,cdatabase,cmeta,automation,reusability,meta,JSON,YAML,python" +)