diff --git a/.github/workflows/check-pr.yml b/.github/workflows/check-pr.yml new file mode 100644 index 0000000..f3f4089 --- /dev/null +++ b/.github/workflows/check-pr.yml @@ -0,0 +1,15 @@ +--- +name: Validate Pull Request +on: + pull_request: +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + +# - name: Lint YAML files +# run: yamllint transformations_apikey/ --format=github + + - name: Lint YAML files (OIDC) + run: yamllint config/transformations/ --format=github diff --git a/.github/workflows/deploy-bootstrap-cli.yaml b/.github/workflows/deploy-bootstrap-cli.yaml new file mode 100644 index 0000000..99ee1e5 --- /dev/null +++ b/.github/workflows/deploy-bootstrap-cli.yaml @@ -0,0 +1,55 @@ +--- +name: deploy-bootstrap-cli + +on: + push: + branches: + #- main + - none + paths: + - config/bootstrap/** + - .github/workflows/deploy-bootstrap-cli.yaml + +jobs: + bootstrap_cdf: + name: Bootstrap CDF + runs-on: ubuntu-latest + strategy: + matrix: + environment: [test, prod] + include: + - environment: test + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: [CDF cluster] + # CDF Project name + CDF_PROJECT: [CDF project] + # Azure AD tenant id + IDP_TENANT: [AAD tenant id] + - environment: prod + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: [CDF cluster] + # CDF Project name + CDF_PROJECT: [CDF project] + # Azure AD tenant id + IDP_TENANT: [AAD tenant id] + + environment: ${{ matrix.environment}} + env: + CDF_HOST: https://${{ matrix.CDF_CLUSTER }}.cognitedata.com/ + + steps: + - uses: actions/checkout@v3 + - uses: cognitedata/inso-bootstrap-cli@v3.0.7 + env: + BOOTSTRAP_IDP_CLIENT_ID: ${{ secrets.CLIENT_ID }} + BOOTSTRAP_CDF_HOST: ${{ env.CDF_HOST }} + BOOTSTRAP_CDF_PROJECT: ${{ matrix.CDF_PROJECT }} + BOOTSTRAP_IDP_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} + BOOTSTRAP_IDP_TOKEN_URL: https://login.microsoftonline.com/${{ matrix.IDP_TENANT }}/oauth2/v2.0/token + BOOTSTRAP_IDP_SCOPES: ${{ env.CDF_HOST }}.default + + with: + # Path to Bootstrap CLI configuration file + config_file: ./config/bootstrap/bootstrap-cli-config.yml + # "yes"|"no" deploy with special groups and aad_mappings + #with_special_groups: "yes" diff --git a/.github/workflows/deploy-extpipes-cli.yaml b/.github/workflows/deploy-extpipes-cli.yaml new file mode 100644 index 0000000..6ae2095 --- /dev/null +++ b/.github/workflows/deploy-extpipes-cli.yaml @@ -0,0 +1,55 @@ +--- +name: deploy-extpipes-cli + +on: + push: + branches: + #- main + - none + paths: + - config/extpipes/** + - .github/workflows/deploy-extpipes-cli.yaml + +jobs: + deploy_extpipes: + name: deploy extpipes-cli + runs-on: ubuntu-latest + strategy: + matrix: + environment: [test, prod] + include: + - environment: test + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: [CDF cluster] + # CDF Project name + CDF_PROJECT: [CDF project] + # Azure AD tenant id + IDP_TENANT: [AAD tenant id] + - environment: prod + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: [CDF cluster] + # CDF Project name + CDF_PROJECT: [CDF project] + # Azure AD tenant id + IDP_TENANT: [AAD tenant id] + + environment: ${{ matrix.environment}} + env: + CDF_HOST: https://${{ matrix.CDF_CLUSTER }}.cognitedata.com/ + + steps: + - uses: actions/checkout@v3 + - uses: cognitedata/inso-extpipes-cli@v2.2.1 + env: + EXTPIPES_IDP_CLIENT_ID: ${{ secrets.CLIENT_ID }} + EXTPIPES_IDP_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} + EXTPIPES_CDF_HOST: ${{ env.CDF_HOST }} + EXTPIPES_CDF_PROJECT: ${{ matrix.CDF_PROJECT }} + EXTPIPES_IDP_TOKEN_URL: https://login.microsoftonline.com/${{ matrix.IDP_TENANT }}/oauth2/v2.0/token + EXTPIPES_IDP_SCOPES: ${{ env.CDF_HOST }}.default + + with: + # Path to extpipes cli configuration file + config_file: ./config/extpipes/extpipes-cli-config.yml + # "yes"|"no" deploy with special groups and aad_mappings + #with_special_groups: "yes" diff --git a/.github/workflows/deploy-tranformations.yml b/.github/workflows/deploy-tranformations.yml new file mode 100644 index 0000000..1a331ea --- /dev/null +++ b/.github/workflows/deploy-tranformations.yml @@ -0,0 +1,54 @@ +--- +name: Deploy transformations + +on: + push: + #branches: [main] + branches: [none] + paths: + - config/transformations/** + - .github/workflows/deploy-transformations.yml + +jobs: + deploy: + name: Deploy transformations + runs-on: ubuntu-latest + strategy: + matrix: + environment: [test, prod] + include: + - environment: test + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: + # CDF Project name + CDF_PROJECT: + # Azure AD tenant id + IDP_TENANT: + - environment: prod + # CDF cluster, like bluefield, greenfield or westeurope-1 + CDF_CLUSTER: + # CDF Project name + CDF_PROJECT: + # Azure AD tenant id + IDP_TENANT: + + environment: ${{ matrix.environment}} + env: + CDF_HOST: https://${{ matrix.CDF_CLUSTER }}.cognitedata.com/ + # Credentials to be used when running your transformations, + # as referenced in your manifests + COGNITE_CLIENT_ID: ${{ secrets.CLIENT_ID }} + COGNITE_CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }} + CDF_PROJECT_NAME: ${{ matrix.CDF_PROJECT }} + + steps: + - uses: actions/checkout@v2 + - uses: cognitedata/transformations-cli@v2.3.9 + with: + path: config/transformations + client-id: ${{ secrets.CLIENT_ID }} + client-secret: ${{ secrets.CLIENT_SECRET }} + cdf-project-name: ${{ matrix.CDF_PROJECT }} + token-url: https://login.microsoftonline.com/${{ matrix.IDP_TENANT }}/oauth2/v2.0/token + scopes: ${{ env.CDF_HOST }}.default + cluster: ${{ matrix.CDF_CLUSTER }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..80ea78d --- /dev/null +++ b/.gitignore @@ -0,0 +1,132 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +#pyCharm +.idea diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..3d29d56 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,24 @@ +--- +repos: + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.21.0 # or higher tag + hooks: + - id: yamllint + args: [--format, parsable] + + - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt + rev: 0.2.1 # or specific tag + hooks: + - id: yamlfmt + args: [--width, '150'] + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: debug-statements + - id: trailing-whitespace + - id: check-yaml + - id: check-builtin-literals + - id: end-of-file-fixer + - id: check-ast + - id: check-docstring-first diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..195336c --- /dev/null +++ b/.yamllint @@ -0,0 +1,9 @@ +--- +extends: default + +rules: + truthy: disable + comments-indentation: disable # don't bother me with this rule + line-length: # don't fail if longer + max: 150 + level: warning diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..977f7a2 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# Enablement Bootstrap +Configuration and transformations for Cognite bootcamp example + +CDF Bootstrap configuration: +https://github.com/cognitedata/inso-bootstrap-cli + +CDF extraction pipelines: +https://github.com/cognitedata/inso-extpipes-cli + +Transformations template: +https://github.com/cognitedata/transformations-action-template diff --git a/config/bootstrap/bootstrap-cli-config.yml b/config/bootstrap/bootstrap-cli-config.yml new file mode 100644 index 0000000..47c603b --- /dev/null +++ b/config/bootstrap/bootstrap-cli-config.yml @@ -0,0 +1,106 @@ +--- +bootstrap: + # This is a template bootstrap CLI file that will need to be completed. For a more complete example look at + # https://github.com/cognitedata/inso-bootstrap-cli/blob/main/configs/config-deploy-example-v2.yml + + features: + # v2 adding as features, available as cli paramaters only atm (v1) + # allowed values are parsed case-insensitive: [true|yes|YES|..] or [false|no|NO|..] + # not as strings in quotes "yes" or 'yes' + + # The default and recommended value is: false + with-datamodel-capability: false + + # default and recommended is: true + with-raw-capability: true + + # new in v2 to configure prior (v1) hard-coded naming elements + # Most of the naming-configurations don't have a functional effect, but affect + # naming of Groups, Datasets and RAWDBs only. + # All given feature values are reflecting the default values, + # to support a smooth v1 migration. + # + # Recommended values for a new v2-project are given as comments! + + + # default is: allprojects + # recommended is shorter: 'all' + aggregated-level-name: all + + # default and recommended is: cdf + # supports empty-string '' + group-prefix: cdf + + # default is: dataset + # recommended is shorter 'ds' or minimal '' + # supports empty-string '' + dataset-suffix: ds + + # default is: rawdb + # recommended is shorter 'db' or minimal '' + rawdb-suffix: db + + # Suffix added to all space names. The recommended value is spc. Defaults to space + space-suffix: spc + + # default is" ['state'] + # meant for separating the statestores from extractors from ingested data + # recommended: keep it or [] + # more rawdbs can be configured at a later stage + # supports empty list [] for no additional variants + rawdb-additional-variants: + # provide more than one rawdb per ns-nodes + # atm (v1) hardcoded is one additional rawdb + - state + + # Mapping between IDP (Azure Active Directory) groups and CDF groups. + idp-cdf-mappings: + # Support multiple CDF Projects, like dev/test/prod in one config. + - cdf-project: # i.e. bootcamp-01-test + mappings: + - cdf-group: cdf:root + idp-source-id: + idp-source-name: + - cdf-group: cdf:all:owner + idp-source-id: # for the AAD Group to be matched to cdf-group + idp-source-name: # i.e. bootcamp-01-test-all-owner + - cdf-group: cdf:all:read + idp-source-id: # for the AAD Group to be matched to cdf-group + idp-source-name: # i.e. bootcamp-01-test-all-read + + + namespaces: + # Typical config will have a namespace for data sources ("src") and a namespace for use cases ("uc") + - ns-name: + description: Data sources for the Ice Cream Factory # HINT + + # Node names will normally follow src:001:srcname or uc:001:usecasename naming convention + ns-nodes: + - node-name: + description: Data from SAP # HINT + # HINT: Add another namespace below + + +# de-facto standards for CDF CogniteClient and logger configuration. +# Following the same parameter structure as the 'Cognite Python Extractor-Utilities' +# and 'Cognite DB Extractor' configuration. +cognite: # kwargs to pass to the CogniteClient, Environment variable format: ${ENVIRONMENT_VARIABLE_NAME} + # host: https://${BOOTSTRAP_CDF_CLUSTER}.cognitedata.com/ + host: ${BOOTSTRAP_CDF_HOST} + project: ${BOOTSTRAP_CDF_PROJECT} + # + # AAD IdP login credentials: + # + idp-authentication: + client-id: ${BOOTSTRAP_IDP_CLIENT_ID} + secret: ${BOOTSTRAP_IDP_CLIENT_SECRET} + scopes: + - ${BOOTSTRAP_IDP_SCOPES} + token_url: ${BOOTSTRAP_IDP_TOKEN_URL} + +logger: + file: + path: ./logs/bootstrap-cli-logs.log + level: INFO + console: + level: INFO diff --git a/config/extpipes/extpipes-cli-config.yml b/config/extpipes/extpipes-cli-config.yml new file mode 100644 index 0000000..4d067e2 --- /dev/null +++ b/config/extpipes/extpipes-cli-config.yml @@ -0,0 +1,51 @@ +--- +# follows the same parameter structure as the DB extractor configuration +cognite: # kwargs to pass to the CogniteClient, Environment variable format: ${ENVIRONMENT_VARIABLE} + # host: https://${EXTPIPES_CDF_CLUSTER}.cognitedata.com/ + host: ${EXTPIPES_CDF_HOST} + project: ${EXTPIPES_CDF_PROJECT} + # + # AAD IdP login credentials: + # + idp-authentication: + client-id: ${EXTPIPES_IDP_CLIENT_ID} + secret: ${EXTPIPES_IDP_CLIENT_SECRET} + scopes: + - ${EXTPIPES_IDP_SCOPES} + token_url: ${EXTPIPES_IDP_TOKEN_URL} + +logger: + file: + path: ./logs/extpipes-deploy.log + level: INFO + console: + level: INFO + +# extpipe-pattern only documentation atm +extpipe-pattern: '{source}:{short-name}:{rawtable-name}:{suffix}' + +# can contain multiple contacts, can be overwritten on pipeline level +default-contacts: + - name: + email: + role: + send-notification: true + +rawdbs: + # list of raw-dbs > containing rawtables > containing pipelines + - rawdb-name: :db + dataset-external-id: :ds + short-name: + rawtables: + - rawtable-name: + pipelines: + # source is a short-name identifying the pipeline source being + # a 'db-extractor (db)', an 'Azure Function (az-func)', + # or 'Azure Data Factory (adf)', 'Python script (py)', .. + - source: + schedule: On trigger + contacts: + - name: + email: + role: + send-notification: false diff --git a/config/transformations/assets_transform_sample/manifest.yaml b/config/transformations/assets_transform_sample/manifest.yaml new file mode 100644 index 0000000..386a00e --- /dev/null +++ b/config/transformations/assets_transform_sample/manifest.yaml @@ -0,0 +1,20 @@ +--- +# Manifest file downloaded from fusion +externalId: tr-asset-sample +name: create_asset_sample +query: + file: transformation.sql +destination: + type: asset_hierarchy +ignoreNullFields: true +shared: true +action: upsert +dataSetExternalId: src:all +# Added authentication information +authentication: + clientId: ${COGNITE_CLIENT_ID} + clientSecret: ${COGNITE_CLIENT_SECRET} + tokenUrl: https://login.microsoftonline.com/16e3985b-ebe8-4e24-9da4-933e21a9fc81/oauth2/v2.0/token + cdfProjectName: ${CDF_PROJECT_NAME} + scopes: + - https://westeurope-1.cognitedata.com/.default diff --git a/config/transformations/assets_transform_sample/transformation.sql b/config/transformations/assets_transform_sample/transformation.sql new file mode 100644 index 0000000..7d55d61 --- /dev/null +++ b/config/transformations/assets_transform_sample/transformation.sql @@ -0,0 +1,9 @@ +SELECT external_id AS externalId, + IF(parent_external_id = '' + OR parent_external_id IS NULL, '', parent_external_id) AS parentExternalId, + name AS name, + to_metadata(metadata) AS metadata, + description AS description, + 'from RAW' AS source, + Dataset_id('src:001:sap:ds') AS dataSetId +FROM `src:001:sap:db`.assets diff --git a/grafana/grafana.json b/grafana/grafana.json new file mode 100644 index 0000000..e69de29 diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..5660e86 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,199 @@ +[[package]] +name = "cfgv" +version = "3.3.1" +description = "Validate configuration and produce human readable error messages." +category = "dev" +optional = false +python-versions = ">=3.6.1" + +[[package]] +name = "distlib" +version = "0.3.6" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "filelock" +version = "3.9.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2022.12.7)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=7.0.1)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "identify" +version = "2.5.13" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +license = ["ukkonen"] + +[[package]] +name = "nodeenv" +version = "1.7.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" + +[package.dependencies] +setuptools = "*" + +[[package]] +name = "platformdirs" +version = "2.6.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.2.2)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] + +[[package]] +name = "pre-commit" +version = "2.21.0" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +virtualenv = ">=20.10.0" + +[[package]] +name = "PyYAML" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "setuptools" +version = "66.0.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "virtualenv" +version = "20.17.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +distlib = ">=0.3.6,<1" +filelock = ">=3.4.1,<4" +platformdirs = ">=2.4,<3" + +[package.extras] +docs = ["proselint (>=0.13)", "sphinx (>=5.3)", "sphinx-argparse (>=0.3.2)", "sphinx-rtd-theme (>=1)", "towncrier (>=22.8)"] +testing = ["coverage (>=6.2)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=21.3)", "pytest (>=7.0.1)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.6.1)", "pytest-randomly (>=3.10.3)", "pytest-timeout (>=2.1)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.8" +content-hash = "02a9d8eeab85a32422072fbd4ff47850d13e66c1229b9b9863ac2aead0e70570" + +[metadata.files] +cfgv = [ + {file = "cfgv-3.3.1-py2.py3-none-any.whl", hash = "sha256:c6a0883f3917a037485059700b9e75da2464e6c27051014ad85ba6aaa5884426"}, + {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, +] +distlib = [ + {file = "distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, + {file = "distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, +] +filelock = [ + {file = "filelock-3.9.0-py3-none-any.whl", hash = "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d"}, + {file = "filelock-3.9.0.tar.gz", hash = "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de"}, +] +identify = [ + {file = "identify-2.5.13-py2.py3-none-any.whl", hash = "sha256:8aa48ce56e38c28b6faa9f261075dea0a942dfbb42b341b4e711896cbb40f3f7"}, + {file = "identify-2.5.13.tar.gz", hash = "sha256:abb546bca6f470228785338a01b539de8a85bbf46491250ae03363956d8ebb10"}, +] +nodeenv = [ + {file = "nodeenv-1.7.0-py2.py3-none-any.whl", hash = "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e"}, + {file = "nodeenv-1.7.0.tar.gz", hash = "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b"}, +] +platformdirs = [ + {file = "platformdirs-2.6.2-py3-none-any.whl", hash = "sha256:83c8f6d04389165de7c9b6f0c682439697887bca0aa2f1c87ef1826be3584490"}, + {file = "platformdirs-2.6.2.tar.gz", hash = "sha256:e1fea1fe471b9ff8332e229df3cb7de4f53eeea4998d3b6bfff542115e998bd2"}, +] +pre-commit = [ + {file = "pre_commit-2.21.0-py2.py3-none-any.whl", hash = "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad"}, + {file = "pre_commit-2.21.0.tar.gz", hash = "sha256:31ef31af7e474a8d8995027fefdfcf509b5c913ff31f2015b4ec4beb26a6f658"}, +] +PyYAML = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] +setuptools = [ + {file = "setuptools-66.0.0-py3-none-any.whl", hash = "sha256:a78d01d1e2c175c474884671dde039962c9d74c7223db7369771fcf6e29ceeab"}, + {file = "setuptools-66.0.0.tar.gz", hash = "sha256:bd6eb2d6722568de6d14b87c44a96fac54b2a45ff5e940e639979a3d1792adb6"}, +] +virtualenv = [ + {file = "virtualenv-20.17.1-py3-none-any.whl", hash = "sha256:ce3b1684d6e1a20a3e5ed36795a97dfc6af29bc3970ca8dab93e11ac6094b3c4"}, + {file = "virtualenv-20.17.1.tar.gz", hash = "sha256:f8b927684efc6f1cc206c9db297a570ab9ad0e51c16fa9e45487d36d1905c058"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..93a41b9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.poetry] +name = "enablement-bootcamp-config" +version = "0.1.0" +description = "" +authors = ["Angela Dang Atkinson "] +readme = "README.md" + +[tool.poetry.dependencies] +python = "^3.8" + + +[tool.poetry.group.dev.dependencies] +pre-commit = "^2.21.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api"