support include yaml

pyproject.toml

@@ -22,6 +22,7 @@ docker = "^4.3.1"
 bech32 = "^1.1.0"
 multitail2 = "^1.5.2"
 python-dotenv = "^0.19.2"
+pyyaml-include = "^1.3"
 
 [tool.poetry.dev-dependencies]
 pytest = "^6.2.5"

pystarport/expansion.py

@@ -2,9 +2,11 @@
 from pathlib import Path
 from typing import Any, Mapping, Optional, Text
 
+import jsonmerge
 import yaml
 from dotenv import dotenv_values, load_dotenv
 from dotenv.variables import parse_variables
+from yamlinclude import YamlIncludeConstructor
 
 
 def expand_posix_vars(obj: Any, variables: Mapping[Text, Optional[Any]]) -> Any:
@@ -43,13 +45,25 @@ def _expand(value, variables):
 
 
 def expand_yaml(config_path, dotenv):
-    config = yaml.safe_load(open(config_path))
+    path = Path(config_path)
+    parent = path.parent
+    YamlIncludeConstructor.add_to_loader_class(
+        loader_class=yaml.FullLoader,
+        base_dir=parent,
+    )
+
+    with open(path) as f:
+        config = yaml.load(f, Loader=yaml.FullLoader)
+
+    include = config.pop("include", {})
+    if include:
+        config = jsonmerge.merge(include, config)
 
     def expand(dotenv):
         if not isinstance(dotenv, str):
             raise ValueError(f"Invalid value passed to dotenv: {dotenv}")
         config_vars = dict(os.environ)  # load system env
-        env_path = Path(config_path).parent.joinpath(dotenv)
+        env_path = parent.joinpath(dotenv)
         if not env_path.is_file():
             raise ValueError(
                 f"Dotenv specified in config but not found at path: {env_path}"

pystarport/tests/test_expansion/base.yaml

@@ -0,0 +1,52 @@
+cronos_777-1:
+  cmd: cronosd
+  start-flags: "--trace"
+  app-config:
+    minimum-gas-prices: 5000000000000basetcro
+    json-rpc:
+      address: "0.0.0.0:{EVMRPC_PORT}"
+      ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
+  validators:
+    - coins: 1000000000000000000stake,10000000000000000000000basetcro
+      staked: 1000000000000000000stake
+      mnemonic: visit craft resemble online window solution west chuckle music diesel vital settle comic tribe project blame bulb armed flower region sausage mercy arrive release
+    - coins: 1000000000000000000stake,10000000000000000000000basetcro
+      staked: 1000000000000000000stake
+      mnemonic: direct travel shrug hand twice agent sail sell jump phone velvet pilot mango charge usual multiply orient garment bleak virtual action mention panda vast
+  accounts:
+    - name: community
+      coins: 10000000000000000000000basetcro
+      mnemonic: "notable error gospel wave pair ugly measure elite toddler cost various fly make eye ketchup despair slab throw tribe swarm word fruit into inmate"
+    - name: signer1
+      coins: 20000000000000000000000basetcro
+      mnemonic: shed crumble dismiss loyal latin million oblige gesture shrug still oxygen custom remove ribbon disorder palace addict again blanket sad flock consider obey popular
+    - name: signer2
+      coins: 30000000000000000000000basetcro
+      mnemonic: night renew tonight dinner shaft scheme domain oppose echo summer broccoli agent face guitar surface belt veteran siren poem alcohol menu custom crunch index
+
+  genesis:
+    consensus_params:
+      block:
+        max_bytes: "1048576"
+        max_gas: "81500000"
+    app_state:
+      evm:
+        params:
+          evm_denom: basetcro
+      cronos:
+        params:
+          cronos_admin: crc12luku6uxehhak02py4rcz65zu0swh7wjsrw0pp
+          enable_auto_deployment: true
+          ibc_cro_denom: ibc/6411AE2ADA1E73DB59DB151A8988F9B7D5E7E233D8414DB6817F8F1A01611F86
+      gov:
+        voting_params:
+          voting_period: "10s"
+        deposit_params:
+          max_deposit_period: "10s"
+          min_deposit:
+            - denom: "basetcro"
+              amount: "1"
+      transfer:
+        params:
+          receive_enabled: true
+          send_enabled: true

pystarport/tests/test_expansion/cronos_has_dotenv.yaml

@@ -1,12 +1,6 @@
+include: !include base.yaml
 dotenv: dotenv
 cronos_777-1:
-  cmd: cronosd
-  start-flags: "--trace"
-  app-config:
-    minimum-gas-prices: 5000000000000basetcro
-    json-rpc:
-      address: "0.0.0.0:{EVMRPC_PORT}"
-      ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
   validators:
     - coins: 1000000000000000000stake,10000000000000000000000basetcro
       staked: 1000000000000000000stake
@@ -26,28 +20,8 @@ cronos_777-1:
       mnemonic: ${SIGNER2_MNEMONIC}
 
   genesis:
-    consensus_params:
-      block:
-        max_bytes: "1048576"
-        max_gas: "81500000"
     app_state:
-      evm:
-        params:
-          evm_denom: basetcro
       cronos:
         params:
           cronos_admin: ${CRONOS_ADMIN}
-          enable_auto_deployment: true
           ibc_cro_denom: ${IBC_CRO_DENOM}
-      gov:
-        voting_params:
-          voting_period: "10s"
-        deposit_params:
-          max_deposit_period: "10s"
-          min_deposit:
-            - denom: "basetcro"
-              amount: "1"
-      transfer:
-        params:
-          receive_enabled: true
-          send_enabled: true

pystarport/tests/test_expansion/cronos_has_posix_no_dotenv.yaml

@@ -1,11 +1,5 @@
+include: !include base.yaml
 cronos_777-1:
-  cmd: cronosd
-  start-flags: "--trace"
-  app-config:
-    minimum-gas-prices: 5000000000000basetcro
-    json-rpc:
-      address: "0.0.0.0:{EVMRPC_PORT}"
-      ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
   validators:
     - coins: 1000000000000000000stake,10000000000000000000000basetcro
       staked: 1000000000000000000stake
@@ -25,28 +19,9 @@ cronos_777-1:
       mnemonic: ${SIGNER2_MNEMONIC}
 
   genesis:
-    consensus_params:
-      block:
-        max_bytes: "1048576"
-        max_gas: "81500000"
     app_state:
-      evm:
-        params:
-          evm_denom: basetcro
       cronos:
         params:
           cronos_admin: ${CRONOS_ADMIN}
-          enable_auto_deployment: true
           ibc_cro_denom: ${IBC_CRO_DENOM}
-      gov:
-        voting_params:
-          voting_period: "10s"
-        deposit_params:
-          max_deposit_period: "10s"
-          min_deposit:
-            - denom: "basetcro"
-              amount: "1"
-      transfer:
-        params:
-          receive_enabled: true
-          send_enabled: true
+

pystarport/tests/test_expansion/cronos_no_dotenv.yaml

@@ -1,52 +1 @@
-cronos_777-1:
-  cmd: cronosd
-  start-flags: "--trace"
-  app-config:
-    minimum-gas-prices: 5000000000000basetcro
-    json-rpc:
-      address: "0.0.0.0:{EVMRPC_PORT}"
-      ws-address: "0.0.0.0:{EVMRPC_PORT_WS}"
-  validators:
-    - coins: 1000000000000000000stake,10000000000000000000000basetcro
-      staked: 1000000000000000000stake
-      mnemonic: visit craft resemble online window solution west chuckle music diesel vital settle comic tribe project blame bulb armed flower region sausage mercy arrive release
-    - coins: 1000000000000000000stake,10000000000000000000000basetcro
-      staked: 1000000000000000000stake
-      mnemonic: direct travel shrug hand twice agent sail sell jump phone velvet pilot mango charge usual multiply orient garment bleak virtual action mention panda vast
-  accounts:
-    - name: community
-      coins: 10000000000000000000000basetcro
-      mnemonic: "notable error gospel wave pair ugly measure elite toddler cost various fly make eye ketchup despair slab throw tribe swarm word fruit into inmate"
-    - name: signer1
-      coins: 20000000000000000000000basetcro
-      mnemonic: shed crumble dismiss loyal latin million oblige gesture shrug still oxygen custom remove ribbon disorder palace addict again blanket sad flock consider obey popular
-    - name: signer2
-      coins: 30000000000000000000000basetcro
-      mnemonic: night renew tonight dinner shaft scheme domain oppose echo summer broccoli agent face guitar surface belt veteran siren poem alcohol menu custom crunch index
-
-  genesis:
-    consensus_params:
-      block:
-        max_bytes: "1048576"
-        max_gas: "81500000"
-    app_state:
-      evm:
-        params:
-          evm_denom: basetcro
-      cronos:
-        params:
-          cronos_admin: crc12luku6uxehhak02py4rcz65zu0swh7wjsrw0pp
-          enable_auto_deployment: true
-          ibc_cro_denom: ibc/6411AE2ADA1E73DB59DB151A8988F9B7D5E7E233D8414DB6817F8F1A01611F86
-      gov:
-        voting_params:
-          voting_period: "10s"
-        deposit_params:
-          max_deposit_period: "10s"
-          min_deposit:
-            - denom: "basetcro"
-              amount: "1"
-      transfer:
-        params:
-          receive_enabled: true
-          send_enabled: true
+include: !include base.yaml

pystarport/tests/test_expansion/test_expansion.py

@@ -8,25 +8,26 @@
 
 
 def test_expansion():
-    cronos_has_dotenv = Path(__file__).parent / "cronos_has_dotenv.yaml"
-    cronos_no_dotenv = Path(__file__).parent / "cronos_no_dotenv.yaml"
-    cronos_has_posix_no_dotenv = (
-        Path(__file__).parent / "cronos_has_posix_no_dotenv.yaml"
-    )
+    parent = Path(__file__).parent
+    base = parent / "base.yaml"
+    cronos_has_dotenv = parent / "cronos_has_dotenv.yaml"
+    cronos_no_dotenv = parent / "cronos_no_dotenv.yaml"
+    cronos_has_posix_no_dotenv = parent / "cronos_has_posix_no_dotenv.yaml"
 
+    baseConfig = yaml.safe_load(open(base))
     # `expand_yaml` is backward compatible, not expanded, and no diff
-    assert yaml.safe_load(open(cronos_no_dotenv)) == expand_yaml(cronos_no_dotenv, None)
+    assert baseConfig == expand_yaml(cronos_no_dotenv, None)
 
     # `expand_yaml` is expanded but no diff
     assert not DeepDiff(
-        yaml.safe_load(open(cronos_no_dotenv)),
+        baseConfig,
         expand_yaml(cronos_has_dotenv, None),
         ignore_order=True,
     )
 
     # overriding dotenv with relative path is expanded and has diff)
     assert DeepDiff(
-        yaml.safe_load(open(cronos_no_dotenv)),
+        baseConfig,
         expand_yaml(cronos_has_dotenv, "dotenv1"),
         ignore_order=True,
     ) == {
@@ -42,7 +43,7 @@ def test_expansion():
 
     # overriding dotenv with absolute path is expanded and has diff
     assert DeepDiff(
-        yaml.safe_load(open(cronos_no_dotenv)),
+        baseConfig,
         expand_yaml(cronos_has_dotenv, os.path.abspath("test_expansion/dotenv1")),
         ignore_order=True,
     ) == {
@@ -58,7 +59,7 @@ def test_expansion():
 
     # overriding dotenv with absolute path is expanded and no diff
     assert not DeepDiff(
-        yaml.safe_load(open(cronos_no_dotenv)),
+        baseConfig,
         expand_yaml(
             cronos_has_posix_no_dotenv, os.path.abspath("test_expansion/dotenv")
         ),