diff --git a/builds.yaml b/builds.yaml new file mode 100644 index 0000000..7174b4c --- /dev/null +++ b/builds.yaml @@ -0,0 +1,43 @@ +--- +project: "Lain's Playground: Sandbox V2" + + +registries: + resim-infra: + account_id: "909785973729" + region: "us-east-1" + auth: + profile: "infrastructure" + +resim_app_config: + client_id: "gTp1Y0kOyQ7QzIo2lZm0auGM6FJZZVvy" + auth_url: "https://resim.us.auth0.com/" + api_url: "https://api.resim.ai/v1/" + +experience_build_configs: + drone_experience: + description: "A drone experience build" + repo: + name: "drone-simulator" + registry: "resim-infra" + version_tag_prefix: "drone_sim_" + system: "Drone Motion Planning System" + branch: auto + version: auto + + build_command: + path: "systems/drone/" + +metrics_build_configs: + drone_metrics: + name: "A drone metrics build" + repo: + name: "drone-simulator" + registry: "resim-infra" + version_tag_prefix: "drone_sim_metrics_" + systems: + - "Drone Motion Planning System" + version: auto + + build_command: + path: "systems/drone/metrics/" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7104a91 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +docker>=7.1.0 +resim-open-core>=0.6.1 +PyYAML>=6.0.2 +boto3>=1.36.10 +GitPython>=3.1.44 +pydantic>=2.10.6 diff --git a/requirements_lock.txt b/requirements_lock.txt new file mode 100644 index 0000000..5a44325 --- /dev/null +++ b/requirements_lock.txt @@ -0,0 +1,103 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --output-file=requirements_lock.txt requirements.txt +# +annotated-types==0.7.0 + # via pydantic +anyio==4.8.0 + # via httpx +attrs==25.1.0 + # via resim-open-core +boto3==1.36.10 + # via -r requirements.txt +botocore==1.36.10 + # via + # boto3 + # s3transfer +certifi==2024.12.14 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.1 + # via requests +docker==7.1.0 + # via -r requirements.txt +exceptiongroup==1.2.2 + # via anyio +gitdb==4.0.12 + # via gitpython +gitpython==3.1.44 + # via -r requirements.txt +h11==0.14.0 + # via httpcore +httpcore==1.0.7 + # via httpx +httpx==0.28.1 + # via resim-open-core +idna==3.10 + # via + # anyio + # httpx + # requests +jmespath==1.0.1 + # via + # boto3 + # botocore +narwhals==1.24.1 + # via plotly +numpy==2.2.2 + # via + # pandas + # resim-open-core +packaging==24.2 + # via plotly +pandas==2.2.3 + # via resim-open-core +plotly==6.0.0 + # via resim-open-core +polling2==0.5.0 + # via resim-open-core +protobuf==5.29.3 + # via resim-open-core +pydantic==2.10.6 + # via -r requirements.txt +pydantic-core==2.27.2 + # via pydantic +python-dateutil==2.9.0.post0 + # via + # botocore + # pandas + # resim-open-core +pytz==2024.2 + # via pandas +pyyaml==6.0.2 + # via -r requirements.txt +requests==2.32.3 + # via + # docker + # resim-open-core +resim-open-core==0.6.1 + # via -r requirements.txt +s3transfer==0.11.2 + # via boto3 +six==1.17.0 + # via python-dateutil +smmap==5.0.2 + # via gitdb +sniffio==1.3.1 + # via anyio +typing-extensions==4.12.2 + # via + # anyio + # pydantic + # pydantic-core +tzdata==2025.1 + # via pandas +urllib3==2.3.0 + # via + # botocore + # docker + # requests diff --git a/scripts/builds_config.py b/scripts/builds_config.py new file mode 100644 index 0000000..bc57a92 --- /dev/null +++ b/scripts/builds_config.py @@ -0,0 +1,53 @@ +from pydantic import BaseModel + + +class ResimAppConfig(BaseModel): + client_id: str + auth_url: str + api_url: str + + +class RegistryAuth(BaseModel): + profile: str + + +class ImageRegistry(BaseModel): + account_id: str + region: str + auth: RegistryAuth + + +class ImageRepo(BaseModel): + name: str + registry: str + + +class BuildCommand(BaseModel): + path: str + + +class ExperienceBuild(BaseModel): + description: str + repo: ImageRepo + version_tag_prefix: str + system: str + branch: str + version: str + build_command: BuildCommand + + +class MetricsBuild(BaseModel): + name: str + repo: ImageRepo + version_tag_prefix: str + systems: list[str] + version: str + build_command: BuildCommand + + +class Builds(BaseModel): + project: str + registries: dict[str, ImageRegistry] + resim_app_config: ResimAppConfig + experience_build_configs: dict[str, ExperienceBuild] + metrics_build_configs: dict[str, MetricsBuild] diff --git a/scripts/create_builds.py b/scripts/create_builds.py new file mode 100755 index 0000000..b803eb3 --- /dev/null +++ b/scripts/create_builds.py @@ -0,0 +1,137 @@ +#!/bin/python + +import argparse +import logging + +import docker +import yaml +from docker.client import DockerClient + +from scripts.utils import ( + get_client, + get_project, + register_experience_build, + register_metrics_build, + get_systems, + get_branches, + docker_ecr_auth, + parse_version, +) + +from scripts.builds_config import Builds, ImageRegistry, MetricsBuild, ExperienceBuild + +logger = logging.getLogger("create_builds") +logger.setLevel(logging.INFO) + + +def list_command(builds, args): + logger.info("Experience Builds:") + for build in builds.experience_build_configs: + logger.info(" %s", build) + + logger.info("Metrics Builds:") + for build in builds.metrics_build_configs: + logger.info(" %s", build) + + +def build_image( + build: ExperienceBuild | MetricsBuild, + registries: dict[str, ImageRegistry], + docker_client: DockerClient, +) -> str: + repo = build.repo + registry = registries[repo.registry] + docker_ecr_auth(docker_client, registry) + command_path = build.build_command.path + full_repo_name = ( + f"{registry.account_id}.dkr.ecr.{registry.region}.amazonaws.com/{repo.name}" + ) + version = parse_version(build.version) + tag = f"{build.version_tag_prefix}{version}" + uri = f"{full_repo_name}:{tag}" + response = docker_client.api.build(path=command_path, tag=uri, decode=True) + for line in response: + logger.info(" ".join((str(v) for v in line.values()))) + + return uri + + +def push_image(uri: str, docker_client: DockerClient): + response = docker_client.api.push(uri, stream=True, decode=True) + for line in response: + logger.info(" ".join((str(v) for v in line.values()))) + + +def build_push(builds, args, *, push: bool): + client = get_client(builds.resim_app_config) + project_id = get_project(builds.project, client).project_id + systems = get_systems(client, project_id) + branches = get_branches(client, project_id) + + docker_client = docker.from_env() + + combined_map = builds.experience_build_configs | builds.metrics_build_configs + for target in args.target_builds: + if target not in builds.experience_build_configs: + continue + build = combined_map[target] + uri = build_image(build, builds.registries, docker_client) + + if not push: + continue + + push_image(uri, docker_client) + register_experience_build(client, project_id, build, uri, systems, branches) + + for target in args.target_builds: + if target not in builds.metrics_build_configs: + continue + + build = combined_map[target] + uri = build_image(build, builds.registries, docker_client) + + if not push: + continue + + push_image(uri, docker_client) + register_metrics_build(client, project_id, build, uri, systems) + + +def push_command(builds, args): + build_push(builds, args, push=True) + + +def build_command(builds, args): + build_push(builds, args, push=False) + + +def main(): + logging.basicConfig() + parser = argparse.ArgumentParser( + prog="create_builds", + description="A simple CLI for building, pushing, and registering builds.", + ) + subparsers = parser.add_subparsers(title="Commands", dest="command", required=True) + + list_parser = subparsers.add_parser("list", help="List all resources") + list_parser.set_defaults(func=list_command) + + # Build command + build_parser = subparsers.add_parser("build", help="Build a selected build") + build_parser.add_argument("target_builds", nargs="*") + build_parser.set_defaults(func=build_command) + + # Push command + push_parser = subparsers.add_parser("push", help="Push a selected build") + push_parser.add_argument("target_builds", nargs="*") + push_parser.set_defaults(func=push_command) + + args = parser.parse_args() + + with open("builds.yaml", "r", encoding="utf-8") as f: + builds = Builds(**yaml.load(f, Loader=yaml.SafeLoader)) + args.func(builds, args) # Call the appropriate function + + +if __name__ == "__main__": + main() diff --git a/scripts/utils.py b/scripts/utils.py new file mode 100644 index 0000000..2fdb968 --- /dev/null +++ b/scripts/utils.py @@ -0,0 +1,157 @@ +import base64 +import boto3 +import git +from docker.client import DockerClient +from resim.auth.python.device_code_client import DeviceCodeClient +from http import HTTPStatus + +from resim.metrics.fetch_all_pages import fetch_all_pages +from resim_python_client.api.builds import create_build_for_branch +from resim_python_client.api.metrics_builds import create_metrics_build + +from resim_python_client.client import AuthenticatedClient +from scripts.builds_config import ImageRegistry, ResimAppConfig, ExperienceBuild, MetricsBuild +from resim.metrics.fetch_all_pages import fetch_all_pages +import logging + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +from resim_python_client.api.projects import ( + create_branch_for_project, + list_branches_for_project, + list_projects, +) +from resim_python_client.api.systems import add_system_to_metrics_build, list_systems +from resim_python_client.client import AuthenticatedClient +from resim_python_client.models import ( + BranchType, + CreateBranchInput, + CreateBuildForBranchInput, + CreateMetricsBuildInput, +) + + +def parse_version(version: str) -> str: + if version == "auto": + repo = git.Repo(search_parent_directories=True) + return repo.head.object.hexsha + return version + + +def parse_branch(branch: str) -> str: + if branch == "auto": + repo = git.Repo(search_parent_directories=True) + return repo.active_branch.name + return branch + + +def docker_ecr_auth(client: DockerClient, registry: ImageRegistry): + session = boto3.Session(profile_name=registry.auth.profile) + ecr_client = session.client("ecr", region_name=registry.region) + token = ecr_client.get_authorization_token() + password = ( + base64.b64decode(token["authorizationData"][0]["authorizationToken"]).decode().split(":")[1] + ) + registry_url = f"{registry.account_id}.dkr.ecr.{registry.region}.amazonaws.com" + client.login(username="AWS", password=password, registry=registry_url) + logger.info("Successfully authenticated to %s.", registry_url) + + +def get_client(config: ResimAppConfig) -> AuthenticatedClient: + auth_client = DeviceCodeClient(domain=config.auth_url, client_id=config.client_id) + token = auth_client.get_jwt()["access_token"] + client = AuthenticatedClient(base_url=config.api_url, token=token) + + return client + + +def get_project(project: str, client: AuthenticatedClient) -> str: + project_pages = fetch_all_pages(list_projects.sync, client=client) + projects = {p.name: p for page in project_pages for p in page.projects} + return projects[project] + + +def get_systems(client: AuthenticatedClient, project_id: str) -> dict[str, str]: + system_pages = fetch_all_pages(list_systems.sync, client=client, project_id=project_id) + return {p.name: p.system_id for page in system_pages for p in page.systems} + + +def get_branches(client: AuthenticatedClient, project_id: str) -> dict[str, str]: + branch_pages = fetch_all_pages( + list_branches_for_project.sync, client=client, project_id=project_id + ) + return {p.name: p.branch_id for page in branch_pages for p in page.branches} + + +def make_branch(client: AuthenticatedClient, project_id: str, branch: str): + response = create_branch_for_project.sync( + project_id=project_id, + client=client, + body=CreateBranchInput(branch_type=BranchType.CHANGE_REQUEST, name=branch), + ) + assert response is not None, "Failed to make branch" + logger.info("Registered branch with id %s", response.branch_id) + + return response.branch_id + + +def register_experience_build( + client: AuthenticatedClient, + project_id: str, + build: ExperienceBuild, + uri: str, + systems: dict[str, str], + branches: dict[str, str], +): + branch = parse_branch(build.branch) + version = parse_version(build.version) + if branch not in branches: + branches[branch] = make_branch(client, project_id, branch) + + response = create_build_for_branch.sync( + project_id=project_id, + branch_id=branches[branch], + client=client, + body=CreateBuildForBranchInput( + image_uri=uri, + system_id=systems[build.system], + version=version, + description=build.description, + ), + ) + assert response is not None + logger.info("Registered experience build with id %s", response.build_id) + + +def register_metrics_build( + client: AuthenticatedClient, + project_id: str, + build: MetricsBuild, + uri: str, + systems: dict[str, str], +): + version = parse_version(build.version) + response = create_metrics_build.sync( + project_id=project_id, + client=client, + body=CreateMetricsBuildInput( + image_uri=uri, + name=build.name, + version=version, + ), + ) + assert response is not None + logger.info("Registered metrics build with id %s", response.metrics_build_id) + metrics_build_id = response.metrics_build_id + + for system in build.systems: + response = add_system_to_metrics_build.sync_detailed( + project_id=project_id, + system_id=systems[system], + metrics_build_id=metrics_build_id, + client=client, + ) + assert response.status_code == HTTPStatus.CREATED, "Failed to add metrics build to system" + logger.info("Added metrics build %s to %s system", metrics_build_id, system)