[WIP] Create Builds From Config File

builds.yaml

@@ -0,0 +1,43 @@
+---
+project: "Lain's Playground: Sandbox V2"
+
+
+registries:
+  resim-infra:
+    account_id: "909785973729"
+    region: "us-east-1"
+    auth:
+      profile: "infrastructure"
+
+resim_app_config:
+  client_id: "gTp1Y0kOyQ7QzIo2lZm0auGM6FJZZVvy"
+  auth_url: "https://resim.us.auth0.com/"
+  api_url: "https://api.resim.ai/v1/"
+
+experience_build_configs:
+  drone_experience:
+    description: "A drone experience build"
+    repo:
+      name: "drone-simulator"
+      registry: "resim-infra"
+    version_tag_prefix: "drone_sim_"
+    system: "Drone Motion Planning System"
+    branch: auto
+    version: auto
+
+    build_command:
+      path: "systems/drone/"
+
+metrics_build_configs:
+  drone_metrics:
+    name: "A drone metrics build"
+    repo:
+      name: "drone-simulator"
+      registry: "resim-infra"
+    version_tag_prefix: "drone_sim_metrics_"
+    systems:
+      - "Drone Motion Planning System"
+    version: auto
+
+    build_command:
+      path: "systems/drone/metrics/"

requirements.txt

@@ -0,0 +1,6 @@
+docker>=7.1.0
+resim-open-core>=0.6.1
+PyYAML>=6.0.2
+boto3>=1.36.10
+GitPython>=3.1.44
+pydantic>=2.10.6

requirements_lock.txt

@@ -0,0 +1,103 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile --output-file=requirements_lock.txt requirements.txt
+#
+annotated-types==0.7.0
+    # via pydantic
+anyio==4.8.0
+    # via httpx
+attrs==25.1.0
+    # via resim-open-core
+boto3==1.36.10
+    # via -r requirements.txt
+botocore==1.36.10
+    # via
+    #   boto3
+    #   s3transfer
+certifi==2024.12.14
+    # via
+    #   httpcore
+    #   httpx
+    #   requests
+charset-normalizer==3.4.1
+    # via requests
+docker==7.1.0
+    # via -r requirements.txt
+exceptiongroup==1.2.2
+    # via anyio
+gitdb==4.0.12
+    # via gitpython
+gitpython==3.1.44
+    # via -r requirements.txt
+h11==0.14.0
+    # via httpcore
+httpcore==1.0.7
+    # via httpx
+httpx==0.28.1
+    # via resim-open-core
+idna==3.10
+    # via
+    #   anyio
+    #   httpx
+    #   requests
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+narwhals==1.24.1
+    # via plotly
+numpy==2.2.2
+    # via
+    #   pandas
+    #   resim-open-core
+packaging==24.2
+    # via plotly
+pandas==2.2.3
+    # via resim-open-core
+plotly==6.0.0
+    # via resim-open-core
+polling2==0.5.0
+    # via resim-open-core
+protobuf==5.29.3
+    # via resim-open-core
+pydantic==2.10.6
+    # via -r requirements.txt
+pydantic-core==2.27.2
+    # via pydantic
+python-dateutil==2.9.0.post0
+    # via
+    #   botocore
+    #   pandas
+    #   resim-open-core
+pytz==2024.2
+    # via pandas
+pyyaml==6.0.2
+    # via -r requirements.txt
+requests==2.32.3
+    # via
+    #   docker
+    #   resim-open-core
+resim-open-core==0.6.1
+    # via -r requirements.txt
+s3transfer==0.11.2
+    # via boto3
+six==1.17.0
+    # via python-dateutil
+smmap==5.0.2
+    # via gitdb
+sniffio==1.3.1
+    # via anyio
+typing-extensions==4.12.2
+    # via
+    #   anyio
+    #   pydantic
+    #   pydantic-core
+tzdata==2025.1
+    # via pandas
+urllib3==2.3.0
+    # via
+    #   botocore
+    #   docker
+    #   requests

scripts/builds_config.py

@@ -0,0 +1,53 @@
+from pydantic import BaseModel
+
+
+class ResimAppConfig(BaseModel):
+    client_id: str
+    auth_url: str
+    api_url: str
+
+
+class RegistryAuth(BaseModel):
+    profile: str
+
+
+class ImageRegistry(BaseModel):
+    account_id: str
+    region: str
+    auth: RegistryAuth
+
+
+class ImageRepo(BaseModel):
+    name: str
+    registry: str
+
+
+class BuildCommand(BaseModel):
+    path: str
+
+
+class ExperienceBuild(BaseModel):
+    description: str
+    repo: ImageRepo
+    version_tag_prefix: str
+    system: str
+    branch: str
+    version: str
+    build_command: BuildCommand
+
+
+class MetricsBuild(BaseModel):
+    name: str
+    repo: ImageRepo
+    version_tag_prefix: str
+    systems: list[str]
+    version: str
+    build_command: BuildCommand
+
+
+class Builds(BaseModel):
+    project: str
+    registries: dict[str, ImageRegistry]
+    resim_app_config: ResimAppConfig
+    experience_build_configs: dict[str, ExperienceBuild]
+    metrics_build_configs: dict[str, MetricsBuild]

scripts/create_builds.py

@@ -0,0 +1,137 @@
+#!/bin/python
+
+import argparse
+import logging
+
+import docker
+import yaml
+from docker.client import DockerClient
+
+from scripts.utils import (
+    get_client,
+    get_project,
+    register_experience_build,
+    register_metrics_build,
+    get_systems,
+    get_branches,
+    docker_ecr_auth,
+    parse_version,
+)
+
+from scripts.builds_config import Builds, ImageRegistry, MetricsBuild, ExperienceBuild
+
+logger = logging.getLogger("create_builds")
+logger.setLevel(logging.INFO)
+
+
+def list_command(builds, args):
+    logger.info("Experience Builds:")
+    for build in builds.experience_build_configs:
+        logger.info("    %s", build)
+
+    logger.info("Metrics Builds:")
+    for build in builds.metrics_build_configs:
+        logger.info("    %s", build)
+
+
+def build_image(
+    build: ExperienceBuild | MetricsBuild,
+    registries: dict[str, ImageRegistry],
+    docker_client: DockerClient,
+) -> str:
+    repo = build.repo
+    registry = registries[repo.registry]
+    docker_ecr_auth(docker_client, registry)
+    command_path = build.build_command.path
+    full_repo_name = (
+        f"{registry.account_id}.dkr.ecr.{registry.region}.amazonaws.com/{repo.name}"
+    )
+    version = parse_version(build.version)
+    tag = f"{build.version_tag_prefix}{version}"
+    uri = f"{full_repo_name}:{tag}"
+    response = docker_client.api.build(path=command_path, tag=uri, decode=True)
+    for line in response:
+        logger.info(" ".join((str(v) for v in line.values())))
+
+    return uri
+
+
+def push_image(uri: str, docker_client: DockerClient):
+    response = docker_client.api.push(uri, stream=True, decode=True)
+    for line in response:
+        logger.info(" ".join((str(v) for v in line.values())))
+
+
+def build_push(builds, args, *, push: bool):
+    client = get_client(builds.resim_app_config)
+    project_id = get_project(builds.project, client).project_id
+    systems = get_systems(client, project_id)
+    branches = get_branches(client, project_id)
+
+    docker_client = docker.from_env()
+
+    combined_map = builds.experience_build_configs | builds.metrics_build_configs
+    for target in args.target_builds:
+        if target not in builds.experience_build_configs:
+            continue
+        build = combined_map[target]
+        uri = build_image(build, builds.registries, docker_client)
+
+        if not push:
+            continue
+
+        push_image(uri, docker_client)
+        register_experience_build(client, project_id, build, uri, systems, branches)
+
+    for target in args.target_builds:
+        if target not in builds.metrics_build_configs:
+            continue
+
+        build = combined_map[target]
+        uri = build_image(build, builds.registries, docker_client)
+
+        if not push:
+            continue
+
+        push_image(uri, docker_client)
+        register_metrics_build(client, project_id, build, uri, systems)
+
+
+def push_command(builds, args):
+    build_push(builds, args, push=True)
+
+
+def build_command(builds, args):
+    build_push(builds, args, push=False)
+
+
+def main():
+    logging.basicConfig()
+    parser = argparse.ArgumentParser(
+        prog="create_builds",
+        description="A simple CLI for building, pushing, and registering builds.",
+    )
+    subparsers = parser.add_subparsers(title="Commands", dest="command", required=True)
+
+    list_parser = subparsers.add_parser("list", help="List all resources")
+    list_parser.set_defaults(func=list_command)
+
+    # Build command
+    build_parser = subparsers.add_parser("build", help="Build a selected build")
+    build_parser.add_argument("target_builds", nargs="*")
+    build_parser.set_defaults(func=build_command)
+
+    # Push command
+    push_parser = subparsers.add_parser("push", help="Push a selected build")
+    push_parser.add_argument("target_builds", nargs="*")
+    push_parser.set_defaults(func=push_command)
+
+    args = parser.parse_args()
+
+    with open("builds.yaml", "r", encoding="utf-8") as f:
+        builds = Builds(**yaml.load(f, Loader=yaml.SafeLoader))
+    args.func(builds, args)  # Call the appropriate function
+
+
+if __name__ == "__main__":
+    main()