diff --git a/.gitignore b/.gitignore index ff52df5..a5b58bb 100644 --- a/.gitignore +++ b/.gitignore @@ -138,4 +138,4 @@ dmypy.json cython_debug/ # VS Code configuration -.vscode \ No newline at end of file +.vscode diff --git a/CHANGELOG.md b/CHANGELOG.md index 35aef4a..0907e1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## UNRELEASED +- [#52](https://github.com/crypto-com/pystarport/pull/52) support jsonnet as config language + *Feb 18, 2022* ## v0.2.4 diff --git a/README.md b/README.md index 58c628c..3eb1d11 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,13 @@ Remember to run `poetry env info` after `poetry install` and update this `python } ``` +## Test +### Install jsonnet +More about [jsonnet](https://jsonnet.org). +``` +make test +``` + ## FAQ diff --git a/poetry.lock b/poetry.lock index 9d14be9..4a306fd 100644 --- a/poetry.lock +++ b/poetry.lock @@ -195,6 +195,14 @@ python-versions = "*" [package.dependencies] jsonschema = "*" +[[package]] +name = "jsonnet" +version = "0.18.0" +description = "Python bindings for Jsonnet - The data templating language" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "jsonschema" version = "3.2.0" @@ -511,7 +519,7 @@ six = "*" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "c2935da305fc294d57633c7822acd89de94ca2d1ffa322d7fc2d32aeb5ab488b" +content-hash = "efa956449bf81ed2943e0187e12995e54ca4bdd088295ee4be0be55232be5a70" [metadata.files] atomicwrites = [ @@ -579,6 +587,9 @@ isort = [ jsonmerge = [ {file = "jsonmerge-1.8.0.tar.gz", hash = "sha256:a86bfc44f32f6a28b749743df8960a4ce1930666b3b73882513825f845cb9558"}, ] +jsonnet = [ + {file = "jsonnet-0.18.0.tar.gz", hash = "sha256:4ccd13427e9097b6b7d6d38f78f638a55ab8b452a257639e8e9af2178ec235d4"}, +] jsonschema = [ {file = "jsonschema-3.2.0-py2.py3-none-any.whl", hash = "sha256:4e5b3cf8216f577bee9ce139cbe72eca3ea4f292ec60928ff24758ce626cd163"}, {file = "jsonschema-3.2.0.tar.gz", hash = "sha256:c8a85b28d377cc7737e46e2d9f2b4f44ee3c0e1deac6bf46ddefc7187d30797a"}, diff --git a/pyproject.toml b/pyproject.toml index b62495a..4e4b5f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ deepdiff = "^5.6.0" flake8 = "^4.0.1" black = "^21.12b0" isort = "^5.10.1" +jsonnet = "^0.18.0" [tool.poetry.scripts] pystarport = "pystarport.cli:main" diff --git a/pystarport/cluster.py b/pystarport/cluster.py index 85ce6b3..9a79f82 100644 --- a/pystarport/cluster.py +++ b/pystarport/cluster.py @@ -24,7 +24,7 @@ from . import ports from .app import CHAIN, IMAGE, SUPERVISOR_CONFIG_FILE from .cosmoscli import ChainCommand, CosmosCLI, ModuleAccount, module_address -from .expansion import expand_yaml +from .expansion import expand_jsonnet, expand_yaml from .ledger import ZEMU_BUTTON_PORT, ZEMU_HOST from .utils import format_doc_string, interact, write_ini @@ -924,7 +924,11 @@ def init_cluster( cmd=None, gen_compose_file=False, ): - config = expand_yaml(config_path, dotenv) + extension = Path(config_path).suffix + if extension == ".jsonnet": + config = expand_jsonnet(config_path, dotenv) + else: + config = expand_yaml(config_path, dotenv) relayer_config = config.pop("relayer", {}) for chain_id, cfg in config.items(): diff --git a/pystarport/expansion.py b/pystarport/expansion.py index 854a57a..d881639 100644 --- a/pystarport/expansion.py +++ b/pystarport/expansion.py @@ -1,7 +1,9 @@ +import json import os from pathlib import Path from typing import Any, Mapping, Optional, Text +import _jsonnet import jsonmerge import yaml from dotenv import dotenv_values, load_dotenv @@ -44,12 +46,34 @@ def _expand(value, variables): return "".join([str(atom.resolve(variables)) for atom in atoms]) +def expand(config, dotenv, path): + config_vars = dict(os.environ) # load system env + + if dotenv is not None: + if "dotenv" in config: + _ = config.pop("dotenv", {}) # remove dotenv field if exists + elif "dotenv" in config: + dotenv = config.pop("dotenv", {}) # pop dotenv field if exists + + if dotenv: + if not isinstance(dotenv, str): + raise ValueError(f"Invalid value passed to dotenv: {dotenv}") + env_path = path.parent / dotenv + if not env_path.is_file(): + raise ValueError( + f"Dotenv specified in config but not found at path: {env_path}" + ) + config_vars.update(dotenv_values(dotenv_path=env_path)) # type: ignore + load_dotenv(dotenv_path=env_path) + + return expand_posix_vars(config, config_vars) + + def expand_yaml(config_path, dotenv): path = Path(config_path) - parent = path.parent YamlIncludeConstructor.add_to_loader_class( loader_class=yaml.FullLoader, - base_dir=parent, + base_dir=path.parent, ) with open(path) as f: @@ -59,26 +83,12 @@ def expand_yaml(config_path, dotenv): if include: config = jsonmerge.merge(include, config) - def expand(dotenv): - if not isinstance(dotenv, str): - raise ValueError(f"Invalid value passed to dotenv: {dotenv}") - config_vars = dict(os.environ) # load system env - env_path = parent.joinpath(dotenv) - if not env_path.is_file(): - raise ValueError( - f"Dotenv specified in config but not found at path: {env_path}" - ) - config_vars.update(dotenv_values(dotenv_path=env_path)) # type: ignore - load_dotenv(dotenv_path=env_path) - return expand_posix_vars(config, config_vars) + config = expand(config, dotenv, path) + return config - if dotenv is not None: - if "dotenv" in config: - _ = config.pop("dotenv", {}) # remove dotenv field if exists - dotenv_path = dotenv - config = expand(dotenv_path) - elif "dotenv" in config: - dotenv_path = config.pop("dotenv", {}) # pop dotenv field if exists - config = expand(dotenv_path) +def expand_jsonnet(config_path, dotenv): + path = Path(config_path) + config = json.loads(_jsonnet.evaluate_file(str(config_path))) + config = expand(config, dotenv, path) return config diff --git a/pystarport/tests/test_expansion/base.jsonnet b/pystarport/tests/test_expansion/base.jsonnet new file mode 100644 index 0000000..55f02be --- /dev/null +++ b/pystarport/tests/test_expansion/base.jsonnet @@ -0,0 +1 @@ +import './default.jsonnet' diff --git a/pystarport/tests/test_expansion/base_yaml.jsonnet b/pystarport/tests/test_expansion/base_yaml.jsonnet new file mode 100644 index 0000000..63ba1e3 --- /dev/null +++ b/pystarport/tests/test_expansion/base_yaml.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc(import './base.jsonnet', true, false) diff --git a/pystarport/tests/test_expansion/cronos_has_dotenv.jsonnet b/pystarport/tests/test_expansion/cronos_has_dotenv.jsonnet new file mode 100644 index 0000000..9b37245 --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_has_dotenv.jsonnet @@ -0,0 +1,40 @@ +local config = import './default.jsonnet'; +local Utils = import 'utils.jsonnet'; + +config { + dotenv+: 'dotenv', + 'cronos_777-1'+: { + validators: [ + Utils.validator('${VALIDATOR1_MNEMONIC}'), + Utils.validator('${VALIDATOR2_MNEMONIC}'), + ], + accounts: [ + Utils.account( + 'community', + '10000000000000000000000basetcro', + '${COMMUNITY_MNEMONIC}', + ), + Utils.account( + 'signer1', + '20000000000000000000000basetcro', + '${SIGNER1_MNEMONIC}', + ), + Utils.account( + 'signer2', + '30000000000000000000000basetcro', + '${SIGNER2_MNEMONIC}', + ), + ], + genesis+: { + app_state+: { + cronos: { + params: { + cronos_admin: '${CRONOS_ADMIN}', + enable_auto_deployment: true, + ibc_cro_denom: '${IBC_CRO_DENOM}', + }, + }, + }, + }, + }, +} diff --git a/pystarport/tests/test_expansion/cronos_has_dotenv_yaml.jsonnet b/pystarport/tests/test_expansion/cronos_has_dotenv_yaml.jsonnet new file mode 100644 index 0000000..beb7aa0 --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_has_dotenv_yaml.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc(import './cronos_has_dotenv.jsonnet', true, false) diff --git a/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv.jsonnet b/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv.jsonnet new file mode 100644 index 0000000..be53a5b --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv.jsonnet @@ -0,0 +1,39 @@ +local config = import './default.jsonnet'; +local Utils = import 'utils.jsonnet'; + +config { + 'cronos_777-1'+: { + validators: [ + Utils.validator('${VALIDATOR1_MNEMONIC}'), + Utils.validator('${VALIDATOR2_MNEMONIC}'), + ], + accounts: [ + Utils.account( + 'community', + '10000000000000000000000basetcro', + '${COMMUNITY_MNEMONIC}', + ), + Utils.account( + 'signer1', + '20000000000000000000000basetcro', + '${SIGNER1_MNEMONIC}', + ), + Utils.account( + 'signer2', + '30000000000000000000000basetcro', + '${SIGNER2_MNEMONIC}', + ), + ], + genesis+: { + app_state+: { + cronos: { + params: { + cronos_admin: '${CRONOS_ADMIN}', + enable_auto_deployment: true, + ibc_cro_denom: '${IBC_CRO_DENOM}', + }, + }, + }, + }, + }, +} diff --git a/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv_yaml.jsonnet b/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv_yaml.jsonnet new file mode 100644 index 0000000..4a8288a --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_has_posix_no_dotenv_yaml.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc(import './cronos_has_posix_no_dotenv.jsonnet', true, false) diff --git a/pystarport/tests/test_expansion/cronos_no_dotenv.jsonnet b/pystarport/tests/test_expansion/cronos_no_dotenv.jsonnet new file mode 100644 index 0000000..55f02be --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_no_dotenv.jsonnet @@ -0,0 +1 @@ +import './default.jsonnet' diff --git a/pystarport/tests/test_expansion/cronos_no_dotenv_yaml.jsonnet b/pystarport/tests/test_expansion/cronos_no_dotenv_yaml.jsonnet new file mode 100644 index 0000000..5467581 --- /dev/null +++ b/pystarport/tests/test_expansion/cronos_no_dotenv_yaml.jsonnet @@ -0,0 +1 @@ +std.manifestYamlDoc(import './cronos_no_dotenv.jsonnet', true, false) diff --git a/pystarport/tests/test_expansion/default.jsonnet b/pystarport/tests/test_expansion/default.jsonnet new file mode 100644 index 0000000..7f79248 --- /dev/null +++ b/pystarport/tests/test_expansion/default.jsonnet @@ -0,0 +1,78 @@ +local Utils = import "utils.jsonnet"; + +{ + 'cronos_777-1': { + cmd: 'cronosd', + 'start-flags': '--trace', + 'app-config': { + 'minimum-gas-prices': '5000000000000basetcro', + 'json-rpc': { + address: '0.0.0.0:{EVMRPC_PORT}', + 'ws-address': '0.0.0.0:{EVMRPC_PORT_WS}', + }, + }, + validators: [ + Utils.validator('visit craft resemble online window solution west chuckle music diesel vital settle comic tribe project blame bulb armed flower region sausage mercy arrive release'), + Utils.validator('direct travel shrug hand twice agent sail sell jump phone velvet pilot mango charge usual multiply orient garment bleak virtual action mention panda vast'), + ], + accounts: [ + Utils.account( + 'community', + '10000000000000000000000basetcro', + 'notable error gospel wave pair ugly measure elite toddler cost various fly make eye ketchup despair slab throw tribe swarm word fruit into inmate', + ), + Utils.account( + 'signer1', + '20000000000000000000000basetcro', + 'shed crumble dismiss loyal latin million oblige gesture shrug still oxygen custom remove ribbon disorder palace addict again blanket sad flock consider obey popular', + ), + Utils.account( + 'signer2', + '30000000000000000000000basetcro', + 'night renew tonight dinner shaft scheme domain oppose echo summer broccoli agent face guitar surface belt veteran siren poem alcohol menu custom crunch index', + ), + ], + genesis: { + consensus_params: { + block: { + max_bytes: '1048576', + max_gas: '81500000', + }, + }, + app_state: { + evm: { + params: { + evm_denom: 'basetcro', + }, + }, + cronos: { + params: { + cronos_admin: 'crc12luku6uxehhak02py4rcz65zu0swh7wjsrw0pp', + enable_auto_deployment: true, + ibc_cro_denom: 'ibc/6411AE2ADA1E73DB59DB151A8988F9B7D5E7E233D8414DB6817F8F1A01611F86', + }, + }, + gov: { + voting_params: { + voting_period: '10s', + }, + deposit_params: { + max_deposit_period: '10s', + min_deposit: [ + { + denom: 'basetcro', + amount: '1', + }, + ], + }, + }, + transfer: { + params: { + receive_enabled: true, + send_enabled: true, + }, + }, + }, + }, + }, +} diff --git a/pystarport/tests/test_expansion/generate-test-yamls b/pystarport/tests/test_expansion/generate-test-yamls new file mode 100755 index 0000000..4c15a39 --- /dev/null +++ b/pystarport/tests/test_expansion/generate-test-yamls @@ -0,0 +1,15 @@ +#!/bin/bash + +set +e +cd "$(dirname "$0")" + +# explicitly set a short TMPDIR to prevent path too long issue on macosx +export TMPDIR=/tmp + +files=("base" "cronos_has_dotenv" "cronos_has_posix_no_dotenv" "cronos_no_dotenv") +length=${#files[@]} +for (( i=0; i<${length}; i++ )) +do + echo "generating "${files[$i]}".yaml" + jsonnet -S ${files[$i]}"_yaml".jsonnet -o ${files[$i]}.yaml +done diff --git a/pystarport/tests/test_expansion/test_expansion.py b/pystarport/tests/test_expansion/test_expansion.py index 234c14b..0f408d1 100644 --- a/pystarport/tests/test_expansion/test_expansion.py +++ b/pystarport/tests/test_expansion/test_expansion.py @@ -1,34 +1,45 @@ import os from pathlib import Path +import pytest import yaml from deepdiff import DeepDiff -from pystarport.expansion import expand_yaml +from pystarport.expansion import expand_jsonnet, expand_yaml -def test_expansion(): - parent = Path(__file__).parent - base = parent / "base.yaml" - cronos_has_dotenv = parent / "cronos_has_dotenv.yaml" - cronos_no_dotenv = parent / "cronos_no_dotenv.yaml" - cronos_has_posix_no_dotenv = parent / "cronos_has_posix_no_dotenv.yaml" +def _get_base_config(): + return yaml.safe_load(open(Path(__file__).parent / "base.yaml")) + - baseConfig = yaml.safe_load(open(base)) +@pytest.mark.parametrize( + "type, func", + [(".yaml", expand_yaml), (".jsonnet", expand_jsonnet)], +) +def test_expansion(type, func): + parent = Path(__file__).parent + cronos_has_dotenv = parent / ("cronos_has_dotenv" + type) + cronos_no_dotenv = parent / ("cronos_no_dotenv" + type) + cronos_has_posix_no_dotenv = parent / ("cronos_no_dotenv" + type) + baseConfig = _get_base_config() # `expand_yaml` is backward compatible, not expanded, and no diff - assert baseConfig == expand_yaml(cronos_no_dotenv, None) + config = func(cronos_no_dotenv, None) + assert baseConfig == config # `expand_yaml` is expanded but no diff + config = func(cronos_has_dotenv, None) assert not DeepDiff( baseConfig, - expand_yaml(cronos_has_dotenv, None), + config, ignore_order=True, ) # overriding dotenv with relative path is expanded and has diff) + dotenv = "dotenv1" + config = func(cronos_has_dotenv, dotenv) assert DeepDiff( baseConfig, - expand_yaml(cronos_has_dotenv, "dotenv1"), + config, ignore_order=True, ) == { "values_changed": { @@ -42,9 +53,11 @@ def test_expansion(): } # overriding dotenv with absolute path is expanded and has diff + dotenv = os.path.abspath("test_expansion/dotenv1") + config = func(cronos_has_dotenv, dotenv) assert DeepDiff( baseConfig, - expand_yaml(cronos_has_dotenv, os.path.abspath("test_expansion/dotenv1")), + config, ignore_order=True, ) == { "values_changed": { @@ -58,10 +71,10 @@ def test_expansion(): } # overriding dotenv with absolute path is expanded and no diff + dotenv = os.path.abspath("test_expansion/dotenv") + config = func(cronos_has_posix_no_dotenv, dotenv) assert not DeepDiff( baseConfig, - expand_yaml( - cronos_has_posix_no_dotenv, os.path.abspath("test_expansion/dotenv") - ), + config, ignore_order=True, ) diff --git a/pystarport/tests/test_expansion/utils.jsonnet b/pystarport/tests/test_expansion/utils.jsonnet new file mode 100644 index 0000000..c844e4e --- /dev/null +++ b/pystarport/tests/test_expansion/utils.jsonnet @@ -0,0 +1,12 @@ +{ + validator(mnemonic):: { + coins: '1000000000000000000stake,10000000000000000000000basetcro', + staked: '1000000000000000000stake', + mnemonic: mnemonic, + }, + account(name, coins, mnemonic):: { + name: name, + coins: coins, + mnemonic: mnemonic, + }, +}