Jsonnet config

.gitignore

@@ -138,4 +138,4 @@ dmypy.json
 cython_debug/
 
 # VS Code configuration
-.vscode
+.vscode

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## UNRELEASED
+- [#52](https://github.com/crypto-com/pystarport/pull/52) support jsonnet as config language
+
 *Feb 18, 2022*
 
 ## v0.2.4

README.md

@@ -251,6 +251,13 @@ Remember to run `poetry env info` after `poetry install` and update this `python
 }
 ```
 
+## Test
+### Install jsonnet
+More about [jsonnet](https://jsonnet.org).
+```
+make test
+```
+
 
 ## FAQ
 

pyproject.toml

@@ -30,6 +30,7 @@ deepdiff = "^5.6.0"
 flake8 = "^4.0.1"
 black = "^21.12b0"
 isort = "^5.10.1"
+jsonnet = "^0.18.0"
 
 [tool.poetry.scripts]
 pystarport = "pystarport.cli:main"

pystarport/cluster.py

@@ -9,7 +9,7 @@
 import sys
 import threading
 import time
-from pathlib import Path
+from pathlib import Path, PurePosixPath
 from typing import List
 
 import durations
@@ -24,7 +24,7 @@
 from . import ports
 from .app import CHAIN, IMAGE, SUPERVISOR_CONFIG_FILE
 from .cosmoscli import ChainCommand, CosmosCLI, ModuleAccount, module_address
-from .expansion import expand_yaml
+from .expansion import expand_jsonnet, expand_yaml
 from .ledger import ZEMU_BUTTON_PORT, ZEMU_HOST
 from .utils import format_doc_string, interact, write_ini
 
@@ -924,7 +924,11 @@ def init_cluster(
     cmd=None,
     gen_compose_file=False,
 ):
-    config = expand_yaml(config_path, dotenv)
+    extension = PurePosixPath(config_path).suffix
+    if extension == ".jsonnet":
+        config = expand_jsonnet(config_path, dotenv)
+    else:
+        config = expand_yaml(config_path, dotenv)
 
     relayer_config = config.pop("relayer", {})
     for chain_id, cfg in config.items():

pystarport/expansion.py

@@ -1,7 +1,9 @@
+import json
 import os
 from pathlib import Path
 from typing import Any, Mapping, Optional, Text
 
+import _jsonnet
 import jsonmerge
 import yaml
 from dotenv import dotenv_values, load_dotenv
@@ -44,12 +46,34 @@ def _expand(value, variables):
     return "".join([str(atom.resolve(variables)) for atom in atoms])
 
 
+def expand(config, dotenv, path):
+    config_vars = dict(os.environ)  # load system env
+
+    if dotenv is not None:
+        if "dotenv" in config:
+            _ = config.pop("dotenv", {})  # remove dotenv field if exists
+    elif "dotenv" in config:
+        dotenv = config.pop("dotenv", {})  # pop dotenv field if exists
+
+    if dotenv:
+        if not isinstance(dotenv, str):
+            raise ValueError(f"Invalid value passed to dotenv: {dotenv}")
+        env_path = path.parent / dotenv
+        if not env_path.is_file():
+            raise ValueError(
+                f"Dotenv specified in config but not found at path: {env_path}"
+            )
+        config_vars.update(dotenv_values(dotenv_path=env_path))  # type: ignore
+        load_dotenv(dotenv_path=env_path)
+
+    return expand_posix_vars(config, config_vars)
+
+
 def expand_yaml(config_path, dotenv):
     path = Path(config_path)
-    parent = path.parent
     YamlIncludeConstructor.add_to_loader_class(
         loader_class=yaml.FullLoader,
-        base_dir=parent,
+        base_dir=path.parent,
     )
 
     with open(path) as f:
@@ -59,26 +83,12 @@ def expand_yaml(config_path, dotenv):
     if include:
         config = jsonmerge.merge(include, config)
 
-    def expand(dotenv):
-        if not isinstance(dotenv, str):
-            raise ValueError(f"Invalid value passed to dotenv: {dotenv}")
-        config_vars = dict(os.environ)  # load system env
-        env_path = parent.joinpath(dotenv)
-        if not env_path.is_file():
-            raise ValueError(
-                f"Dotenv specified in config but not found at path: {env_path}"
-            )
-        config_vars.update(dotenv_values(dotenv_path=env_path))  # type: ignore
-        load_dotenv(dotenv_path=env_path)
-        return expand_posix_vars(config, config_vars)
+    config = expand(config, dotenv, path)
+    return config
 
-    if dotenv is not None:
-        if "dotenv" in config:
-            _ = config.pop("dotenv", {})  # remove dotenv field if exists
-        dotenv_path = dotenv
-        config = expand(dotenv_path)
-    elif "dotenv" in config:
-        dotenv_path = config.pop("dotenv", {})  # pop dotenv field if exists
-        config = expand(dotenv_path)
 
+def expand_jsonnet(config_path, dotenv):
+    path = Path(config_path)
+    config = json.loads(_jsonnet.evaluate_file(str(config_path)))
+    config = expand(config, dotenv, path)
     return config

pystarport/tests/test_expansion/base.jsonnet

@@ -0,0 +1 @@
+import './default.jsonnet'

pystarport/tests/test_expansion/base_yaml.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc(import './base.jsonnet', true, false)

pystarport/tests/test_expansion/cronos_has_dotenv.jsonnet

@@ -0,0 +1,40 @@
+local config = import './default.jsonnet';
+local Utils = import 'utils.jsonnet';
+
+config {
+  dotenv+: 'dotenv',
+  'cronos_777-1'+: {
+    validators: [
+      Utils.validator('${VALIDATOR1_MNEMONIC}'),
+      Utils.validator('${VALIDATOR2_MNEMONIC}'),
+    ],
+    accounts: [
+      Utils.account(
+        'community',
+        '10000000000000000000000basetcro',
+        '${COMMUNITY_MNEMONIC}',
+      ),
+      Utils.account(
+        'signer1',
+        '20000000000000000000000basetcro',
+        '${SIGNER1_MNEMONIC}',
+      ),
+      Utils.account(
+        'signer2',
+        '30000000000000000000000basetcro',
+        '${SIGNER2_MNEMONIC}',
+      ),
+    ],
+    genesis+: {
+      app_state+: {
+        cronos: {
+          params: {
+            cronos_admin: '${CRONOS_ADMIN}',
+            enable_auto_deployment: true,
+            ibc_cro_denom: '${IBC_CRO_DENOM}',
+          },
+        },
+      },
+    },
+  },
+}

pystarport/tests/test_expansion/cronos_has_dotenv_yaml.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc(import './cronos_has_dotenv.jsonnet', true, false)

pystarport/tests/test_expansion/cronos_has_posix_no_dotenv.jsonnet

@@ -0,0 +1,39 @@
+local config = import './default.jsonnet';
+local Utils = import 'utils.jsonnet';
+
+config {
+  'cronos_777-1'+: {
+    validators: [
+      Utils.validator('${VALIDATOR1_MNEMONIC}'),
+      Utils.validator('${VALIDATOR2_MNEMONIC}'),
+    ],
+    accounts: [
+      Utils.account(
+        'community',
+        '10000000000000000000000basetcro',
+        '${COMMUNITY_MNEMONIC}',
+      ),
+      Utils.account(
+        'signer1',
+        '20000000000000000000000basetcro',
+        '${SIGNER1_MNEMONIC}',
+      ),
+      Utils.account(
+        'signer2',
+        '30000000000000000000000basetcro',
+        '${SIGNER2_MNEMONIC}',
+      ),
+    ],
+    genesis+: {
+      app_state+: {
+        cronos: {
+          params: {
+            cronos_admin: '${CRONOS_ADMIN}',
+            enable_auto_deployment: true,
+            ibc_cro_denom: '${IBC_CRO_DENOM}',
+          },
+        },
+      },
+    },
+  },
+}

pystarport/tests/test_expansion/cronos_has_posix_no_dotenv_yaml.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc(import './cronos_has_posix_no_dotenv.jsonnet', true, false)

pystarport/tests/test_expansion/cronos_no_dotenv.jsonnet

@@ -0,0 +1 @@
+import './default.jsonnet'

pystarport/tests/test_expansion/cronos_no_dotenv_yaml.jsonnet

@@ -0,0 +1 @@
+std.manifestYamlDoc(import './cronos_no_dotenv.jsonnet', true, false)

pystarport/tests/test_expansion/default.jsonnet

@@ -0,0 +1,78 @@
+local Utils = import "utils.jsonnet";
+
+{
+  'cronos_777-1': {
+    cmd: 'cronosd',
+    'start-flags': '--trace',
+    'app-config': {
+      'minimum-gas-prices': '5000000000000basetcro',
+      'json-rpc': {
+        address: '0.0.0.0:{EVMRPC_PORT}',
+        'ws-address': '0.0.0.0:{EVMRPC_PORT_WS}',
+      },
+    },
+    validators: [
+      Utils.validator('visit craft resemble online window solution west chuckle music diesel vital settle comic tribe project blame bulb armed flower region sausage mercy arrive release'),
+      Utils.validator('direct travel shrug hand twice agent sail sell jump phone velvet pilot mango charge usual multiply orient garment bleak virtual action mention panda vast'),
+    ],
+    accounts: [
+      Utils.account(
+        'community',
+        '10000000000000000000000basetcro',
+        'notable error gospel wave pair ugly measure elite toddler cost various fly make eye ketchup despair slab throw tribe swarm word fruit into inmate',
+      ),
+      Utils.account(
+        'signer1',
+        '20000000000000000000000basetcro',
+        'shed crumble dismiss loyal latin million oblige gesture shrug still oxygen custom remove ribbon disorder palace addict again blanket sad flock consider obey popular',
+      ),
+      Utils.account(
+        'signer2',
+        '30000000000000000000000basetcro',
+        'night renew tonight dinner shaft scheme domain oppose echo summer broccoli agent face guitar surface belt veteran siren poem alcohol menu custom crunch index',
+      ),
+    ],
+    genesis: {
+      consensus_params: {
+        block: {
+          max_bytes: '1048576',
+          max_gas: '81500000',
+        },
+      },
+      app_state: {
+        evm: {
+          params: {
+            evm_denom: 'basetcro',
+          },
+        },
+        cronos: {
+          params: {
+            cronos_admin: 'crc12luku6uxehhak02py4rcz65zu0swh7wjsrw0pp',
+            enable_auto_deployment: true,
+            ibc_cro_denom: 'ibc/6411AE2ADA1E73DB59DB151A8988F9B7D5E7E233D8414DB6817F8F1A01611F86',
+          },
+        },
+        gov: {
+          voting_params: {
+            voting_period: '10s',
+          },
+          deposit_params: {
+            max_deposit_period: '10s',
+            min_deposit: [
+              {
+                denom: 'basetcro',
+                amount: '1',
+              },
+            ],
+          },
+        },
+        transfer: {
+          params: {
+            receive_enabled: true,
+            send_enabled: true,
+          },
+        },
+      },
+    },
+  },
+}

pystarport/tests/test_expansion/generate-test-yamls

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set +e
+cd "$(dirname "$0")"
+
+# explicitly set a short TMPDIR to prevent path too long issue on macosx
+export TMPDIR=/tmp
+
+files=("base" "cronos_has_dotenv" "cronos_has_posix_no_dotenv" "cronos_no_dotenv")
+length=${#files[@]}
+for (( i=0; i<${length}; i++ ))
+do
+    echo "generating "${files[$i]}".yaml"
+    jsonnet -S ${files[$i]}"_yaml".jsonnet -o ${files[$i]}.yaml
+done