Reorganize imports

bioimageio/spec/__init__.py

@@ -1,4 +1,23 @@
 from . import v0_1, v0_3
-from .latest import *
+from .build_spec import build_spec
 
-__version__ = v0_3.schema.get_args(FormatVersion)[-1]
+# autogen: start
+from bioimageio.spec.shared import fields
+from . import nodes, raw_nodes, schema, utils
+from .converters import maybe_convert_manifest, maybe_convert_model
+from .raw_nodes import FormatVersion
+
+fields = fields
+
+
+get_nn_instance = utils.get_nn_instance
+download_uri_to_local_path = utils.download_uri_to_local_path
+load_raw_model = utils.load_raw_model
+load_model = utils.load_model
+
+# autogen: stop
+
+# assuming schema will always be part of spec
+from bioimageio.spec.shared.common import get_args
+from .raw_nodes import FormatVersion
+__version__ = get_args(FormatVersion)[-1]

bioimageio/spec/converters.py

@@ -0,0 +1,3 @@
+# Auto-generated by generate_passthrough_modules.py - do not modify
+
+from .v0_3.converters import *

bioimageio/spec/nodes.py

@@ -0,0 +1,3 @@
+# Auto-generated by generate_passthrough_modules.py - do not modify
+
+from .v0_3.nodes import *

bioimageio/spec/raw_nodes.py

@@ -0,0 +1,3 @@
+# Auto-generated by generate_passthrough_modules.py - do not modify
+
+from .v0_3.raw_nodes import *

bioimageio/spec/schema.py

@@ -0,0 +1,3 @@
+# Auto-generated by generate_passthrough_modules.py - do not modify
+
+from .v0_3.schema import *

bioimageio/spec/shared/common.py

@@ -3,6 +3,12 @@
 
 from ruamel.yaml import YAML
 
+try:
+    from typing import Literal, get_args
+except ImportError:
+    from typing_extensions import Literal, get_args  # noqa
+
+
 yaml = YAML(typ="safe")
 
 BIOIMAGEIO_CACHE_PATH = pathlib.Path(os.getenv("BIOIMAGEIO_CACHE_PATH", pathlib.Path.home() / "bioimageio_cache"))

bioimageio/spec/shared/raw_nodes.py

@@ -9,6 +9,7 @@
 except ImportError:
     from typing_extensions import get_args, get_origin
 
+from .common import get_args
 from marshmallow import missing
 
 

bioimageio/spec/utils.py

@@ -0,0 +1,3 @@
+# Auto-generated by generate_passthrough_modules.py - do not modify
+
+from .v0_3.utils import *

bioimageio/spec/v0_3/raw_nodes.py

@@ -15,11 +15,7 @@
     URI,
 )
 
-try:
-    from typing import Literal, get_args
-except ImportError:
-    from typing_extensions import Literal, get_args
-
+from bioimageio.spec.shared.common import Literal, get_args
 
 # Ideally only the current format version is valid.
 # Older formats may be converter through `bioimageio.spec.utils.maybe_convert`,

bioimageio/spec/v0_3/schema.py

@@ -6,14 +6,10 @@
 from spdx_license_list import LICENSES
 
 from bioimageio.spec.shared import field_validators, fields
+from bioimageio.spec.shared.common import get_args
 from bioimageio.spec.shared.schema import SharedBioImageIOSchema
 from . import raw_nodes
 
-try:
-    from typing import get_args
-except ImportError:
-    from typing_extensions import get_args
-
 
 class BioImageIOSchema(SharedBioImageIOSchema):
     raw_nodes = raw_nodes

bioimageio/spec/v0_3/utils.py

@@ -7,14 +7,11 @@
 from marshmallow import ValidationError
 
 from bioimageio.spec.shared import BIOIMAGEIO_CACHE_PATH, yaml
+from bioimageio.spec.shared.common import get_args
 from bioimageio.spec.shared.transformers import download_uri_to_local_path, resolve_raw_node_to_node, resolve_uri
 from . import nodes, raw_nodes, schema
 from .converters import maybe_convert_model
 
-try:
-    from typing import get_args
-except ImportError:
-    from typing_extensions import get_args
 
 download_uri_node_to_local_path = download_uri_to_local_path
 

scripts/generate_passthrough_modules.py

@@ -0,0 +1,146 @@
+from pathlib import Path
+from argparse import ArgumentParser
+import re
+import sys
+
+_script_path = Path(__file__).parent
+
+autogen_header = "# Auto-generated by generate_passthrough_modules.py - do not modify\n\n"
+autogen_text = autogen_header + "from .{spec_version}.{stem} import *\n"
+
+
+def get_config(args):
+    return {
+        "main_module_path": (_script_path.parent / "bioimageio" / "spec").resolve(),
+        "versioned_module_path": (_script_path.parent / "bioimageio" / "spec" / args.target_version).resolve(),
+        "target_version": args.target_version,
+    }
+
+
+def remove_autogen_mods(config):
+    for f in config["main_module_path"].glob("*.py"):
+        mod_txt = f.read_text()
+        m = re.match(autogen_header + "from \.(?P<version>v[\d_]+)\.", mod_txt)
+        if m:
+            print(f"Deleting {f} (linked version {m.groupdict()['version']})")
+            f.unlink()
+
+
+def updated_init_content(config) -> str:
+    restr = "# autogen: start\n.*# autogen: stop"
+
+    init_file = config["main_module_path"] / "__init__.py"
+    assert init_file.exists()
+    versioned_init = config["versioned_module_path"] / "__init__.py"
+    module_init = config["main_module_path"] / "__init__.py"
+    vx_init = module_init.read_text()
+    if not re.findall(restr, vx_init, flags=re.DOTALL):
+        raise RuntimeError(
+            f"Could not find autogen markers in {module_init}. Excpected to find\n\n# autogen: start\n...\n# autogen: stop\n\nin your __init__."
+        )
+    return re.sub(restr, f"# autogen: start\n{versioned_init.read_text()}\n# autogen: stop", vx_init, flags=re.DOTALL)
+
+
+def update_init(config):
+    module_init = config["main_module_path"] / "__init__.py"
+    module_init.write_text(updated_init_content(config))
+
+
+def add_autogen_mods(config):
+    for f in config["versioned_module_path"].glob("*.py"):
+        if f.name.startswith("__"):
+            continue
+
+        tmp = config["main_module_path"] / f.name
+        tmp.write_text(autogen_text.format(spec_version=config["target_version"], stem=f.stem))
+
+
+def is_valid_generated_module(module_file: Path, spec_version: str):
+    module_txt = module_file.read_text()
+    if module_txt == autogen_text.format(spec_version=spec_version, stem=module_file.stem):
+        return True
+
+    return False
+
+
+def check_main(args) -> int:
+    print(f"Checking `bioimageio.spec` modules to link against {args.target_version}.")
+    config = get_config(args)
+    print(
+        f"Assuming module location {config['main_module_path']}, with target spec in {config['versioned_module_path']}."
+    )
+
+    ret = 0
+    for f in config["versioned_module_path"].glob("*.py"):
+        if f.name == "__init__.py":
+            continue
+        if not (config["main_module_path"] / f.name).exists() or not is_valid_generated_module(
+            config["main_module_path"] / f.name, config["target_version"]
+        ):
+            ret += 1
+            print(f"Could not find {config['main_module_path'] / f.name}")
+
+    if ret == 0:
+        print("All seems fine.")
+    else:
+        print("Issues found, try regenerating.")
+    return ret
+
+
+def generate_main(args) -> int:
+    print(f"Generating `bioimageio.spec` modules to link against {args.target_version}.")
+
+    config = get_config(args)
+    remove_autogen_mods(config)
+    add_autogen_mods(config)
+    update_init(config)
+
+    return 0
+
+
+def parse_args():
+    p = ArgumentParser(
+        description=(
+            "script that generates Python module files in bioimageio.spec that "
+            "'link' to a certain spec version. The generated modules act as pass"
+            "-through, via `from .vX_Y import *"
+        ),
+    )
+    sps = p.add_subparsers(
+        title="subcommands",
+    )  # help='additional help')
+    p_ck = sps.add_parser("check")
+    p_ck.set_defaults(func=check_main)
+    p_gen = sps.add_parser("generate")
+    p_gen.set_defaults(func=generate_main)
+
+    p_ck.add_argument(
+        "--target-version",
+        required=True,
+        help=("Name of the submodule. This submodule will be made available in `bioimageio.spec`. Example 'v0_3'."),
+        type=str,
+    )
+
+    p_gen.add_argument(
+        "--target-version",
+        required=True,
+        help=("Name of the submodule. This submodule will be made available in `bioimageio.spec`. Example 'v0_3'."),
+        type=str,
+    )
+    p_gen.add_argument(
+        "--dry-run",
+        action="store_true",
+        default=False,
+    )
+
+    args = p.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    return args.func(args)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/test_build_spec.py

@@ -6,7 +6,7 @@
 
 
 def test_build_spec_pickle(rf_config_path):
-    from bioimageio.spec.latest.build_spec import build_spec
+    from bioimageio.spec.build_spec import build_spec
 
     source = yaml.load(rf_config_path)
     source = maybe_convert_model(source)
@@ -49,7 +49,7 @@ def test_build_spec_pickle(rf_config_path):
 
 
 def test_build_spec_pytorch(UNet2DNucleiBroad_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(UNet2DNucleiBroad_model_url)
     assert os.path.exists(config_path), config_path
@@ -87,7 +87,7 @@ def test_build_spec_pytorch(UNet2DNucleiBroad_model_url):
 
 
 def test_build_spec_onnx(UNet2DNucleiBroad_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(UNet2DNucleiBroad_model_url)
     assert os.path.exists(config_path), config_path
@@ -126,7 +126,7 @@ def test_build_spec_onnx(UNet2DNucleiBroad_model_url):
 
 
 def test_build_spec_torchscript(UNet2DNucleiBroad_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(UNet2DNucleiBroad_model_url)
     assert os.path.exists(config_path), config_path
@@ -165,7 +165,7 @@ def test_build_spec_torchscript(UNet2DNucleiBroad_model_url):
 
 
 def test_build_spec_keras(FruNet_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(FruNet_model_url)
     assert os.path.exists(config_path), config_path
@@ -197,7 +197,7 @@ def test_build_spec_keras(FruNet_model_url):
 
 
 def test_build_spec_tf(FruNet_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(FruNet_model_url)
     assert os.path.exists(config_path), config_path
@@ -229,7 +229,7 @@ def test_build_spec_tf(FruNet_model_url):
 
 
 def test_build_spec_tfjs(FruNet_model_url):
-    from bioimageio.spec.latest.build_spec import _get_local_path, build_spec
+    from bioimageio.spec.build_spec import _get_local_path, build_spec
 
     config_path = _get_local_path(FruNet_model_url)
     assert os.path.exists(config_path), config_path