✨ Comp backend: allow multiple amis/bootscripts/pre-pull images (#5033)

ITISFoundation · Nov 16, 2023 · 163d47f · 163d47f
1 parent 5477e3d
commit 163d47f
Show file tree

Hide file tree

Showing 31 changed files with 626 additions and 268 deletions.
diff --git a/.env-devel b/.env-devel
@@ -18,6 +18,12 @@ AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
 
 API_SERVER_DEV_FEATURES_ENABLED=0
 
+AUTOSCALING_DASK=null
+AUTOSCALING_EC2_ACCESS=null
+AUTOSCALING_EC2_INSTANCES=null
+AUTOSCALING_NODES_MONITORING=null
+AUTOSCALING_POLL_INTERVAL=10
+
 BF_API_KEY=none
 BF_API_SECRET=none
 

diff --git a/.pylintrc b/.pylintrc
@@ -243,7 +243,7 @@ contextmanager-decorators=contextlib.contextmanager
 # List of members which are set dynamically and missed by pylint inference
 # system, and so shouldn't trigger E1101 when accessed. Python regular
 # expressions are accepted.
-generated-members=
+generated-members=sh
 
 # Tells whether to warn about missing members when the owner of the attribute
 # is inferred to be None.

diff --git a/packages/aws-library/requirements/_base.in b/packages/aws-library/requirements/_base.in
@@ -10,3 +10,4 @@ aioboto3
 aiocache
 pydantic[email]
 types-aiobotocore[ec2]
+sh
diff --git a/packages/aws-library/requirements/_base.txt b/packages/aws-library/requirements/_base.txt
@@ -154,6 +154,8 @@ rpds-py==0.12.0
     #   referencing
 s3transfer==0.7.0
     # via boto3
+sh==2.0.6
+    # via -r requirements/_base.in
 six==1.16.0
     # via python-dateutil
 tenacity==8.2.3

diff --git a/packages/aws-library/requirements/ci.txt b/packages/aws-library/requirements/ci.txt
@@ -13,6 +13,7 @@
 
 # installs this repo's packages
 ../pytest-simcore/
+../models-library/
 ../service-library/
 ../settings-library/
 

diff --git a/packages/aws-library/requirements/dev.txt b/packages/aws-library/requirements/dev.txt
@@ -13,6 +13,7 @@
 
 # installs this repo's packages
 --editable ../pytest-simcore/
+--editable ../models-library/
 --editable ../service-library/
 --editable ../settings-library/
 

diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py
@@ -1,8 +1,19 @@
 import datetime
+import tempfile
 from dataclasses import dataclass
-from typing import TypeAlias
-
-from pydantic import BaseModel, ByteSize, NonNegativeFloat, PositiveInt
+from typing import Any, ClassVar, TypeAlias
+
+import sh
+from models_library.docker import DockerGenericTag
+from pydantic import (
+    BaseModel,
+    ByteSize,
+    Extra,
+    Field,
+    NonNegativeFloat,
+    PositiveInt,
+    validator,
+)
 from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType
 
 
@@ -74,3 +85,88 @@ class EC2InstanceConfig:
     key_name: str
     security_group_ids: list[str]
     subnet_id: str
+
+
+AMIIdStr: TypeAlias = str
+CommandStr: TypeAlias = str
+
+
+class EC2InstanceBootSpecific(BaseModel):
+    ami_id: AMIIdStr
+    custom_boot_scripts: list[CommandStr] = Field(
+        default_factory=list,
+        description="script(s) to run on EC2 instance startup (be careful!), "
+        "each entry is run one after the other using '&&' operator",
+    )
+    pre_pull_images: list[DockerGenericTag] = Field(
+        default_factory=list,
+        description="a list of docker image/tags to pull on instance cold start",
+    )
+    pre_pull_images_cron_interval: datetime.timedelta = Field(
+        default=datetime.timedelta(minutes=30),
+        description="time interval between pulls of images (minimum is 1 minute) "
+        "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
+    )
+
+    class Config:
+        extra = Extra.forbid
+        schema_extra: ClassVar[dict[str, Any]] = {
+            "examples": [
+                {
+                    # just AMI
+                    "ami_id": "ami-123456789abcdef",
+                },
+                {
+                    # AMI + scripts
+                    "ami_id": "ami-123456789abcdef",
+                    "custom_boot_scripts": ["ls -tlah", "echo blahblah"],
+                },
+                {
+                    # AMI + scripts + pre-pull
+                    "ami_id": "ami-123456789abcdef",
+                    "custom_boot_scripts": ["ls -tlah", "echo blahblah"],
+                    "pre_pull_images": [
+                        "nginx:latest",
+                        "itisfoundation/my-very-nice-service:latest",
+                        "simcore/services/dynamic/another-nice-one:2.4.5",
+                        "asd",
+                    ],
+                },
+                {
+                    # AMI + pre-pull
+                    "ami_id": "ami-123456789abcdef",
+                    "pre_pull_images": [
+                        "nginx:latest",
+                        "itisfoundation/my-very-nice-service:latest",
+                        "simcore/services/dynamic/another-nice-one:2.4.5",
+                        "asd",
+                    ],
+                },
+                {
+                    # AMI + pre-pull + cron
+                    "ami_id": "ami-123456789abcdef",
+                    "pre_pull_images": [
+                        "nginx:latest",
+                        "itisfoundation/my-very-nice-service:latest",
+                        "simcore/services/dynamic/another-nice-one:2.4.5",
+                        "asd",
+                    ],
+                    "pre_pull_images_cron_interval": "01:00:00",
+                },
+            ]
+        }
+
+    @validator("custom_boot_scripts")
+    @classmethod
+    def validate_bash_calls(cls, v):
+        try:
+            with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file:
+                temp_file.writelines(v)
+                temp_file.flush()
+                # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes
+                sh.bash("-n", temp_file.name)
+        except sh.ErrorReturnCode as exc:
+            msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}"
+            raise ValueError(msg) from exc
+
+        return v
diff --git a/packages/settings-library/src/settings_library/ec2.py b/packages/settings-library/src/settings_library/ec2.py
@@ -18,7 +18,7 @@ class Config(BaseCustomSettings.Config):
             "examples": [
                 {
                     "EC2_ACCESS_KEY_ID": "my_access_key_id",
-                    "EC2_ENDPOINT": "http://my_ec2_endpoint.com",
+                    "EC2_ENDPOINT": "https://my_ec2_endpoint.com",
                     "EC2_REGION_NAME": "us-east-1",
                     "EC2_SECRET_ACCESS_KEY": "my_secret_access_key",
                 }

diff --git a/services/autoscaling/README.md b/services/autoscaling/README.md
@@ -1,16 +1,13 @@
 # autoscaling
 
-[![image-size]](https://microbadger.com/images/itisfoundation/autoscaling. "More on itisfoundation/autoscaling.:staging-latest image")
+Service to auto-scale swarm for both dynamic and computational services
 
-[![image-badge]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry")
-[![image-version]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry")
-[![image-commit]](https://microbadger.com/images/itisfoundation/autoscaling "More on Auto scaling service image in registry")
 
-Service to auto-scale swarm
+## development
 
-<!-- Add badges urls here-->
-[image-size]:https://img.shields.io/microbadger/image-size/itisfoundation/autoscaling./staging-latest.svg?label=autoscaling.&style=flat
-[image-badge]:https://images.microbadger.com/badges/image/itisfoundation/autoscaling.svg
-[image-version]https://images.microbadger.com/badges/version/itisfoundation/autoscaling.svg
-[image-commit]:https://images.microbadger.com/badges/commit/itisfoundation/autoscaling.svg
-<!------------------------->
+```
+make install-dev
+make test-dev-unit
+
+# NOTE: there are manual tests that need access to AWS EC2 instances!
+```