Semi-autogenerated cli reference

content/docs/command-reference/_generate_cli_spec.py

@@ -0,0 +1,123 @@
+import json
+from typing import Dict, List, Optional
+
+from click import Command, Context, Group, Option
+from pydantic import BaseModel
+from typer.main import get_group
+
+from mlem import cli
+
+use_group = ["deployment"]
+skip = ["dev"]
+
+abc_group = ["apply-remote", "build", "declare", "serve"]
+
+
+class Opt(BaseModel):
+    decl: str
+    help: str
+
+
+class Args(BaseModel):
+    args: List[Opt]
+    impls: Optional[List[str]]
+    impl_metavar: Optional[str]
+    subcommands: Optional[Dict[str, str]]
+
+
+class Spec(BaseModel):
+    args: Args
+    options: List[Opt]
+    usage: str
+    doc: str
+
+
+def get_options(command: Command, ctx):
+    if command.name not in abc_group:
+        yield from command.get_params(ctx)
+        return
+
+    options = None
+    for subcommand in command.commands.values():
+        if options is None:
+            options = list(get_options(subcommand, ctx))
+            continue
+        new_options = {o.help for o in
+                       get_options(subcommand, ctx)}
+        options = [o for o in options if o.help in new_options]
+    yield from options
+
+
+def repr_option(option, ctx):
+    decl, help = option.get_help_record(ctx)
+    help = help.replace("  ", " ")  # TODO: maybe fix in typer code?
+    return Opt(decl=decl, help=help)
+
+
+def generate_options(command: Command, ctx):
+    res = []
+    for option in get_options(command, ctx):
+        if not isinstance(option, Option):
+            continue
+        res.append(repr_option(option, ctx))
+    return res
+
+
+def generate_args(command, ctx):
+    args = []
+    for arg in command.get_params(ctx):
+        if isinstance(arg, Option):
+            continue
+        args.append(repr_option(arg, ctx))
+    impls = None
+    metavar = None
+    subcommands = None
+    if command.name in abc_group:
+        impls = list(sorted(command.commands))
+        metavar = command.subcommand_metavar
+        args.extend(generate_args(list(command.commands.values())[0], ctx).args)
+    if command.name in use_group:
+        subcommands = {c.name: c.get_short_help_str() for c in
+                       command.commands.values()}
+    return Args(args=args, impls=impls, impl_metavar=metavar,
+                subcommands=subcommands)
+
+def generate_usage(command: Command, ctx):
+    if command.name not in abc_group:
+        return command.get_usage(ctx)
+    subcommand = list(command.commands.values())[0]
+    subctx = Context(subcommand, parent=ctx, info_name=subcommand.name)
+    sub_usage = generate_usage(subcommand, subctx)
+    return sub_usage.replace(subcommand.name, command.subcommand_metavar)
+
+def generate_cli_command(command: Command, ctx):
+    return Spec(args=generate_args(command, ctx),
+                options=generate_options(command, ctx),
+                usage=generate_usage(command, ctx),
+                doc=command.help.strip())
+
+
+def main():
+    group = get_group(cli.app)
+    ctx = Context(group, info_name="mlem")
+    spec = {}
+    for name, command in group.commands.items():
+        if name in skip:
+            continue
+        subctx = Context(command, ctx, info_name=name)
+        if isinstance(command, Group) and name in use_group:
+
+            spec[f"{name}/index"] = generate_cli_command(command, subctx)
+            for subname, subcommand in command.commands.items():
+                subsubctx = Context(subcommand, subctx, info_name=subname)
+                spec[f"{name}/{subname}"] = generate_cli_command(subcommand,
+                                                                 subsubctx)
+            continue
+        spec[name] = generate_cli_command(command, subctx)
+
+    with open("spec.json", "w", encoding="utf8") as f:
+        json.dump({k: v.dict() for k, v in spec.items()}, f, indent=2)
+
+
+if __name__ == '__main__':
+    main()

content/docs/command-reference/_generate_options.py

@@ -0,0 +1,91 @@
+import json
+import re
+from typing import Dict, List
+
+from pydantic import BaseModel, parse_obj_as
+
+from _generate_cli_spec import Args, Opt, Spec
+
+DOC_AUTO_REPLACE = {
+    "MLEM Object": "[MLEM Object](/doc/user-guide/basic-concepts#mlem-objects)"
+}
+
+
+def replace_section(data: str, section_name: str, new_value: str,
+                    section_prefix: str = "## ") -> str:
+    return re.sub(f"{section_prefix}{section_name}(.*?)^{section_prefix}",
+                  f"{section_prefix}{section_name}{new_value}{section_prefix}",
+                  data, flags=re.MULTILINE | re.DOTALL)
+
+
+def repr_option(option: Opt):
+    return f"- `{option.decl}`: {option.help}"
+
+
+def generate_options(options: List[Opt]):
+    res = ["", ""]
+    for option in options:
+        res.append(repr_option(option))
+    return "\n".join(res + ["", ""])
+
+
+def generate_usage(usage, argspec: Args):
+    if argspec.args:
+        args = '\n'.join(repr_option(a) for a in argspec.args)
+        args = f"\n\nArguments:\n{args}"
+    else:
+        args = ""
+    if argspec.impls:
+        impls = "\n".join(f"- {c}" for c in argspec.impls)
+        impls = f"\n\nBuiltin {argspec.impl_metavar}s:\n{impls}"
+    else:
+        impls = ""
+    if argspec.subcommands:
+        subcommands = "\n".join(
+            f"- {k}: {v}" for k, v in argspec.subcommands.items())
+        subcommands = f"\n\nSubcommands:\n{subcommands}"
+    else:
+        subcommands = ""
+    return f"\n{usage}{subcommands}{impls}{args}\n"
+
+
+def generate_doc(doc):
+    for k, v in DOC_AUTO_REPLACE.items():
+        doc = doc.replace(k, v)
+    return f"\n\n{doc}\n\n"
+
+
+def generate_cli_command(name: str, spec: Spec):
+    with open(f"{name}.md", "r", encoding="utf8") as f:
+        data = f.read()
+
+    data = replace_section(data, "usage", generate_usage(spec.usage, spec.args),
+                           section_prefix="```")
+    data = replace_section(data, "Options", generate_options(spec.options))
+
+    cmd_name = name.replace("/", " ")
+    if cmd_name.endswith(" index"):
+        cmd_name = cmd_name[:-len(" index")]
+    data = replace_section(data, " " + cmd_name, generate_doc(spec.doc),
+                           section_prefix="#")
+    with open(f"{name}.md", "w", encoding="utf8") as f:
+        f.write(data)
+
+
+class AllSpec(BaseModel):
+    __root__: Dict[str, Spec]
+
+
+def main():
+    with open("spec.json", "r", encoding="utf8") as f:
+        spec = parse_obj_as(AllSpec, json.load(f))
+
+    for k, s in spec.__root__.items():
+        generate_cli_command(k, s)
+
+
+if __name__ == '__main__':
+    from _generate_cli_spec import main as spec_main
+
+    spec_main()
+    main()

content/docs/command-reference/apply-remote.md

@@ -7,11 +7,14 @@ a MLEM object to `output` if provided. Otherwise, it will be printed to
 ## Synopsis
 
 ```usage
-usage: mlem apply-remote [options] [subtype] data
+Usage: mlem apply-remote client [options] data
 
-arguments:
-[SUBTYPE]  Type of client. Choices: ['http', 'rmq']
-DATA       Path to dataset object  [required]
+Builtin clients:
+- http
+- rmq
+
+Arguments:
+- `DATA`: Path to data object [required]
 ```
 
 ## Description
@@ -35,10 +38,8 @@ clients are `http` and `rmq` - which are used to launch requests against the
 - `--index / --no-index`: Whether to index output in .mlem directory
 - `--json`: Output as json
 - `-l, --load TEXT`: File to load client config from
-- `-c, --conf TEXT`: Options for client in format `field.name=value`
-- `-f, --file_conf TEXT`: File with options for client in format
-  `field.name=path_to_config`
-- `-h, --help`: Show this message and exit.
+- `-f, --file_conf TEXT`: File with options for client in format `field.name=path_to_config`
+- `--help`: Show this message and exit.
 
 ## Example: Apply a locally hosted model to a local dataset
 

content/docs/command-reference/apply.md

@@ -6,11 +6,11 @@ provided. Otherwise, it will be printed to `stdout`.
 ## Synopsis
 
 ```usage
-usage: mlem apply [options] model data
+Usage: mlem apply [options] model data
 
-arguments:
-MODEL  Path to model object  [required]
-DATA   Path to dataset object  [required]
+Arguments:
+- `model`: Path to model object [required]
+- `data`: Path to data object [required]
 ```
 
 ## Description
@@ -36,13 +36,12 @@ datasets.
 - `--data-project, --dr TEXT`: Project with data
 - `--data-rev TEXT`: Revision of data
 - `-i, --import`: Try to import data on-the-fly
-- `--import-type, --it TEXT`: Specify how to read data file for import.
-  Available types: ['pandas', 'pickle']
+- `--import-type, --it TEXT`: Specify how to read data file for import. Available types: ['pandas', 'pickle']
 - `-b, --batch_size INTEGER`: Batch size for reading data in batches.
 - `--index / --no-index`: Whether to index output in .mlem directory
 - `-e, --external`: Save result not in .mlem, but directly in project
 - `--json`: Output as json
-- `-h, --help`: Show this message and exit.
+- `--help`: Show this message and exit.
 
 ## Examples
 

content/docs/command-reference/build.md

@@ -6,11 +6,16 @@ Python package.
 ## Synopsis
 
 ```usage
-usage: mlem build [options] model [subtype]
+Usage: mlem build builder [options] model
 
-arguments:
-MODEL      Path to model  [required]
-[SUBTYPE]  Type of build. Choices: ['whl', 'pip', 'docker_dir', 'docker']
+Builtin builders:
+- docker
+- docker_dir
+- pip
+- whl
+
+Arguments:
+- `MODEL`: Path to model [required]
 ```
 
 ## Description
@@ -24,10 +29,8 @@ images.
 - `-p, --project TEXT`: Path to MLEM project [default: (none)]
 - `--rev TEXT`: Repo revision to use [default: (none)]
 - `-l, --load TEXT`: File to load builder config from
-- `-c, --conf TEXT`: Options for builder in format `field.name=value`
-- `-f, --file_conf TEXT`: File with options for builder in format
-  `field.name=path_to_config`
-- `-h, --help`: Show this message and exit.
+- `-f, --file_conf TEXT`: File with options for builder in format `field.name=path_to_config`
+- `--help`: Show this message and exit.
 
 ## Examples
 

content/docs/command-reference/checkenv.md

@@ -0,0 +1,11 @@
+# checkenv
+
+Check that current environment satisfies object requrements
+
+## Options
+
+- `-p, --project TEXT`: Path to MLEM project [default: (none)]
+- `--rev TEXT`: Repo revision to use [default: (none)]
+- `--help`: Show this message and exit.
+
+## Examples

content/docs/command-reference/clone.md

@@ -6,11 +6,11 @@ saves a copy of it to `target` path.
 ## Synopsis
 
 ```usage
-usage: mlem clone [options] uri target
+Usage: mlem clone [options] uri target
 
-arguments:
-URI     URI to object you want to clone  [required]
-TARGET  Path to store the downloaded object.  [required]
+Arguments:
+- `URI`: URI to object you want to clone [required]
+- `TARGET`: Path to store the downloaded object. [required]
 ```
 
 ## Description
@@ -26,7 +26,7 @@ repository.
 - `--rev TEXT`: Repo revision to use [default: (none)]
 - `--target-project, --tp TEXT`: Project to save target to [default: (none)]
 - `-e, --external`: Save result not in .mlem, but directly in project
-- `--link / --no-link`: Whether to create link for output in .mlem directory
+- `--index / --no-index`: Whether to index output in .mlem directory
 - `--help`: Show this message and exit.
 
 ## Examples

content/docs/command-reference/config.md

@@ -0,0 +1,16 @@
+# config
+
+Manipulate MLEM configuration
+
+## Synopsys
+
+
+```usage
+Usage: mlem config [options] command [args]...
+```
+
+## Options
+
+- `--help`: Show this message and exit.
+
+## Examples