Skip to content

Commit

Permalink
new docker label dictionary
Browse files Browse the repository at this point in the history
  • Loading branch information
@sanderegg
sanderegg committed Jul 4, 2023
1 parent 2858082 commit 9f60948
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 53 deletions.
127 changes: 107 additions & 20 deletions packages/models-library/src/models_library/docker.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import contextlib
import re
from typing import Any, Final

Expand All @@ -6,9 +7,18 @@
from models_library.projects import ProjectID
from models_library.projects_nodes import NodeID
from models_library.users import UserID
from pydantic import BaseModel, ByteSize, ConstrainedStr, Field, root_validator
from pydantic import (
BaseModel,
ByteSize,
ConstrainedStr,
Field,
ValidationError,
parse_obj_as,
root_validator,
)

from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX
from .utils.fastapi_encoders import jsonable_encoder


class DockerLabelKey(ConstrainedStr):
Expand All @@ -24,14 +34,18 @@ class DockerGenericTag(ConstrainedStr):

_SIMCORE_CONTAINER_PREFIX: Final[str] = "io.simcore.container."
_BACKWARDS_COMPATIBILITY_MAP: Final[dict[str, str]] = {
"user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id",
"study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
"project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
"uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
"node_id": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
"product_name": f"{_SIMCORE_CONTAINER_PREFIX}product-name",
"project_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
"simcore_user_agent": f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent",
"study_id": f"{_SIMCORE_CONTAINER_PREFIX}project-id",
"user_id": f"{_SIMCORE_CONTAINER_PREFIX}user-id",
"uuid": f"{_SIMCORE_CONTAINER_PREFIX}node-id",
"mem_limit": f"{_SIMCORE_CONTAINER_PREFIX}memory-limit",
"swarm_stack_name": f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name",
}
_UNDEFINED_VALUE_STR: Final[str] = "undefined"
_UNDEFINED_VALUE_INT: Final[str] = "0"


class SimcoreServiceDockerLabelKeys(BaseModel):
Expand All @@ -46,36 +60,52 @@ class SimcoreServiceDockerLabelKeys(BaseModel):
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent"
)

# None is for backwards compatibility, can be removed in a few sprints
memory_limit: ByteSize | None = Field(
swarm_stack_name: str = Field(
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}swarm-stack-name"
)

memory_limit: ByteSize = Field(
..., alias=f"{_SIMCORE_CONTAINER_PREFIX}memory-limit"
)
cpu_limit: float | None = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit")
cpu_limit: float = Field(..., alias=f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit")

@root_validator(pre=True)
def _backwards_compatibility(cls, values: dict[str, Any]) -> dict[str, Any]:
# NOTE: this is necessary for deployment and legacy service
# NOTE: this is necessary for dy-sidecar and legacy service until they are adjusted
if mapped_values := {
_BACKWARDS_COMPATIBILITY_MAP[k]: v
for k, v in values.items()
if k in _BACKWARDS_COMPATIBILITY_MAP
}:
# these values were sometimes omitted, so let's provide some defaults
for key in ["product-name", "simcore-user-agent", "swarm-stack-name"]:
mapped_values.setdefault(
f"{_SIMCORE_CONTAINER_PREFIX}{key}", _UNDEFINED_VALUE_STR
)

mapped_values.setdefault(
f"{_SIMCORE_CONTAINER_PREFIX}product-name", "osparc"
f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", _UNDEFINED_VALUE_INT
)

def _convert_nano_cpus_to_cpus(nano_cpu: str) -> str:
with contextlib.suppress(ValidationError):
return f"{parse_obj_as(float, nano_cpu) / (1.0*10**9):.2f}"
return _UNDEFINED_VALUE_INT

mapped_values.setdefault(
f"{_SIMCORE_CONTAINER_PREFIX}simcore-user-agent", "undefined"
f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit",
_convert_nano_cpus_to_cpus(
values.get("nano_cpus_limit", _UNDEFINED_VALUE_INT)
),
)
mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}memory-limit", "0")
mapped_values.setdefault(f"{_SIMCORE_CONTAINER_PREFIX}cpu-limit", "0")
return mapped_values
return values

def to_docker_labels(self) -> dict[DockerLabelKey, str]:
"""returns a dictionary of strings as required by docker"""
std_export = self.dict(by_alias=True)
std_export = jsonable_encoder(self, by_alias=True)
return {
DockerLabelKey(f"{_SIMCORE_CONTAINER_PREFIX}{k.replace('_', '-')}"): f"{v}"
DockerLabelKey(f"{k.replace('_', '-')}"): f"{v}"
for k, v in sorted(std_export.items())
}

Expand All @@ -90,23 +120,80 @@ class Config:
allow_population_by_field_name = True
schema_extra = {
"examples": [
# legacy with no limits (a.k.a all dynamic services)
# legacy service labels
{
"port": "8080", # TODO: go away
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"swarm_stack_name": "devel-simcore",
"type": "main", # TODO: go away
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
},
# legacy container labels
{
"mem_limit": "1073741824",
"nano_cpus_limit": "4000000000",
"node_id": "1f963626-66e1-43f1-a777-33955c08b909",
"simcore_user_agent": "puppeteer",
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"swarm_stack_name": "devel-simcore",
"user_id": "5",
},
# dy-sidecar service labels
{
"key": "simcore/services/dynamic/jupyter-math", # TODO: go away
"port": "8888", # TODO: go away
"service_image": "itisfoundation/dynamic-sidecar:master-github-latest", # TODO: go away
"service_port": "8888", # TODO: go away
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"swarm_stack_name": "devel-simcore",
"type": "main-v2", # TODO: go away
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
"version": "2.0.9", # TODO: go away
# "simcore_user_agent": "???????misssing??"
},
# dy-sidecar container labels
{
"mem_limit": "1073741824",
"nano_cpus_limit": "4000000000",
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
},
# dy-proxy service labels
{
"dynamic-type": "dynamic-sidecar",
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"swarm_stack_name": "devel-simcore",
"type": "dependency-v2",
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
},
# dy-proxy container labels
{
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
},
# dy-sidecar user-services labels
{
"product_name": "osparc",
"simcore_user_agent": "puppeteer",
"study_id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"user_id": "5",
"uuid": "1f963626-66e1-43f1-a777-33955c08b909",
},
# modern both dynamic-sidecar services and computational services
{
"io.simcore.container.cpu-limit": "2.4",
"io.simcore.container.memory-limit": "1073741824",
"io.simcore.container.node-id": "1f963626-66e1-43f1-a777-33955c08b909",
"io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"io.simcore.container.user-id": "5",
"io.simcore.container.product-name": "osparc",
"io.simcore.container.project-id": "29f393fc-1410-47b3-b4b9-61dfce21a2a6",
"io.simcore.container.simcore-user-agent": "puppeteer",
"io.simcore.container.memory-limit": "1073741824",
"io.simcore.container.cpu-limit": "2.4",
"io.simcore.container.swarm-stack-name": "devel-osparc",
"io.simcore.container.user-id": "5",
},
]
}
44 changes: 11 additions & 33 deletions packages/models-library/tests/test_docker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import pytest
from faker import Faker
from models_library.docker import (
_SIMCORE_CONTAINER_PREFIX,
DockerGenericTag,
DockerLabelKey,
SimcoreServiceDockerLabelKeys,
Expand Down Expand Up @@ -106,42 +107,19 @@ def test_docker_generic_tag(image_name: str, valid: bool):


@pytest.mark.parametrize(
"obj_data",
[
pytest.param(
{
"user_id": _faker.pyint(),
"project_id": _faker.uuid4(),
"node_id": _faker.uuid4(),
},
id="parse_existing_service_labels-and-legacy",
),
pytest.param(
{
"user_id": _faker.pyint(),
"project_id": _faker.uuid4(),
"node_id": _faker.uuid4(),
"product_name": "test_p",
"simcore_user_agent": "a-test-puppet",
},
id="parse_new_service_labels-soon-to-removed",
),
pytest.param(
{
"io.simcore.container.user_id": _faker.pyint(),
"io.simcore.container.project_id": _faker.uuid4(),
"io.simcore.container.node_id": _faker.uuid4(),
"io.simcore.container.product-name": "test_p",
"io.simcore.container.simcore_user_agent": "a-test-puppet",
},
id="labels following docker rules",
),
],
"obj_data", SimcoreServiceDockerLabelKeys.Config.schema_extra["examples"], ids=str
)
def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]):
simcore_service_docker_label_keys = SimcoreServiceDockerLabelKeys.parse_obj(
obj_data
)
exported_dict = simcore_service_docker_label_keys.to_docker_labels()
assert all(isinstance(v, str) for v in exported_dict.values())
assert parse_obj_as(SimcoreServiceDockerLabelKeys, exported_dict)
assert all(
isinstance(v, str) for v in exported_dict.values()
), "docker labels must be strings!"
assert all(key.startswith(_SIMCORE_CONTAINER_PREFIX) for key in exported_dict)
re_imported_docker_label_keys = parse_obj_as(
SimcoreServiceDockerLabelKeys, exported_dict
)
assert re_imported_docker_label_keys
assert simcore_service_docker_label_keys == re_imported_docker_label_keys

0 comments on commit 9f60948

Please sign in to comment.