From a8d1240ac41c61974be8233f6a0115c130e8cd9d Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Wed, 6 Dec 2023 17:42:29 +0100
Subject: [PATCH 01/13] adding new setting

---
 .../src/simcore_service_clusters_keeper/core/settings.py     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
index 107e87391b6..34337e8dd4e 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
@@ -129,6 +129,11 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings):
         " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html),"
         "this is required to start a new EC2 instance",
     )
+    PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: dict[str, str] = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
 
     @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES")
     @classmethod

From 033ea61e290005a72b1ffaf6ea240fd640becb7c Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 16:28:47 +0100
Subject: [PATCH 02/13] added types for ec2 tag key and values

---
 packages/aws-library/src/aws_library/ec2/client.py | 11 ++++++++---
 packages/aws-library/src/aws_library/ec2/models.py | 14 +++++++++++++-
 packages/aws-library/tests/test_ec2_models.py      | 14 ++++++++++++--
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/packages/aws-library/src/aws_library/ec2/client.py b/packages/aws-library/src/aws_library/ec2/client.py
index a88cf93cbd7..f4b5436bee9 100644
--- a/packages/aws-library/src/aws_library/ec2/client.py
+++ b/packages/aws-library/src/aws_library/ec2/client.py
@@ -7,7 +7,7 @@
 import botocore.exceptions
 from aiobotocore.session import ClientCreatorContext
 from aiocache import cached
-from pydantic import ByteSize
+from pydantic import ByteSize, parse_obj_as
 from servicelib.logging_utils import log_context
 from settings_library.ec2 import EC2Settings
 from types_aiobotocore_ec2 import EC2Client
@@ -170,7 +170,9 @@ async def start_aws_instance(
                     else None,
                     type=instance["InstanceType"],
                     state=instance["State"]["Name"],
-                    tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                    tags=parse_obj_as(
+                        EC2Tags, {tag["Key"]: tag["Value"] for tag in instance["Tags"]}
+                    ),
                     resources=Resources(
                         cpus=instance_config.type.cpus, ram=instance_config.type.ram
                     ),
@@ -236,7 +238,10 @@ async def get_instances(
                             cpus=ec2_instance_types[0].cpus,
                             ram=ec2_instance_types[0].ram,
                         ),
-                        tags={tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                        tags=parse_obj_as(
+                            EC2Tags,
+                            {tag["Key"]: tag["Value"] for tag in instance["Tags"]},
+                        ),
                     )
                 )
         _logger.debug(
diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py
index 21b0f89b513..26ec502d70b 100644
--- a/packages/aws-library/src/aws_library/ec2/models.py
+++ b/packages/aws-library/src/aws_library/ec2/models.py
@@ -1,4 +1,5 @@
 import datetime
+import re
 import tempfile
 from dataclasses import dataclass
 from typing import Any, ClassVar, TypeAlias
@@ -8,6 +9,7 @@
 from pydantic import (
     BaseModel,
     ByteSize,
+    ConstrainedStr,
     Extra,
     Field,
     NonNegativeFloat,
@@ -60,7 +62,17 @@ class EC2InstanceType:
 
 
 InstancePrivateDNSName: TypeAlias = str
-EC2Tags: TypeAlias = dict[str, str]
+
+
+class AWSTagKey(ConstrainedStr):
+    regex = re.compile(r"^(?!.*(\.{1,2}|_index))[a-zA-Z0-9\+\-=\._:@]{1,128}$")
+
+
+class AWSTagValue(ConstrainedStr):
+    regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\._:/@]{0,256}$")
+
+
+EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue]
 
 
 @dataclass(frozen=True)
diff --git a/packages/aws-library/tests/test_ec2_models.py b/packages/aws-library/tests/test_ec2_models.py
index aac4a1d7863..847cd306e44 100644
--- a/packages/aws-library/tests/test_ec2_models.py
+++ b/packages/aws-library/tests/test_ec2_models.py
@@ -4,8 +4,8 @@
 
 
 import pytest
-from aws_library.ec2.models import Resources
-from pydantic import ByteSize
+from aws_library.ec2.models import AWSTagKey, AWSTagValue, Resources
+from pydantic import ByteSize, ValidationError, parse_obj_as
 
 
 @pytest.mark.parametrize(
@@ -122,3 +122,13 @@ def test_resources_sub(a: Resources, b: Resources, result: Resources):
     assert a - b == result
     a -= b
     assert a == result
+
+
+@pytest.mark.parametrize("ec2_tag_key", ["", "/", " ", ".", "..", "_index"])
+def test_aws_tag_key_invalid(ec2_tag_key: str):
+    # for a key it raises
+    with pytest.raises(ValidationError):
+        parse_obj_as(AWSTagKey, ec2_tag_key)
+
+    # for a value it does not
+    parse_obj_as(AWSTagValue, ec2_tag_key)

From a742b21ca573ebec3c2199d268195e5208063a08 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 16:31:49 +0100
Subject: [PATCH 03/13] use aws tag key/value class

---
 .../modules/auto_scaling_mode_base.py         |  4 +-
 .../auto_scaling_mode_computational.py        |  4 +-
 .../modules/auto_scaling_mode_dynamic.py      |  4 +-
 .../utils/utils_ec2.py                        | 40 +++++++++++++------
 .../utils/ec2.py                              | 29 +++++++++-----
 5 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py
index 3c9e2742075..d375a511a5e 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_base.py
@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import DockerLabelKey
 from models_library.generated_models.docker_rest_api import Node as DockerNode
@@ -24,7 +24,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[DockerNode]:
 
     @staticmethod
     @abstractmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         ...
 
     @staticmethod
diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py
index 9ddb7df5c14..8ad53e676e9 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_computational.py
@@ -2,7 +2,7 @@
 import logging
 from collections.abc import Iterable
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import (
     DOCKER_TASK_EC2_INSTANCE_TYPE_PLACEMENT_CONSTRAINT_KEY,
@@ -42,7 +42,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[Node]:
         return await utils_docker.get_worker_nodes(get_docker_client(app))
 
     @staticmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         app_settings = get_application_settings(app)
         return utils_ec2.get_ec2_tags_computational(app_settings)
 
diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py
index ec3870eb422..3e1737b814c 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_mode_dynamic.py
@@ -1,6 +1,6 @@
 from collections.abc import Iterable
 
-from aws_library.ec2.models import EC2InstanceData, Resources
+from aws_library.ec2.models import EC2InstanceData, EC2Tags, Resources
 from fastapi import FastAPI
 from models_library.docker import DockerLabelKey
 from models_library.generated_models.docker_rest_api import Node, Task
@@ -31,7 +31,7 @@ async def get_monitored_nodes(app: FastAPI) -> list[Node]:
         )
 
     @staticmethod
-    def get_ec2_tags(app: FastAPI) -> dict[str, str]:
+    def get_ec2_tags(app: FastAPI) -> EC2Tags:
         app_settings = get_application_settings(app)
         return utils_ec2.get_ec2_tags_dynamic(app_settings)
 
diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
index 88c18314460..e167a36fd9c 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
@@ -8,7 +8,13 @@
 from collections.abc import Callable
 from textwrap import dedent
 
-from aws_library.ec2.models import EC2InstanceType, Resources
+from aws_library.ec2.models import (
+    AWSTagKey,
+    AWSTagValue,
+    EC2InstanceType,
+    EC2Tags,
+    Resources,
+)
 
 from .._meta import VERSION
 from ..core.errors import ConfigurationError, Ec2InstanceNotFoundError
@@ -17,32 +23,40 @@
 logger = logging.getLogger(__name__)
 
 
-def get_ec2_tags_dynamic(app_settings: ApplicationSettings) -> dict[str, str]:
+def get_ec2_tags_dynamic(app_settings: ApplicationSettings) -> EC2Tags:
     assert app_settings.AUTOSCALING_NODES_MONITORING  # nosec
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
     return {
-        "io.simcore.autoscaling.version": f"{VERSION}",
-        "io.simcore.autoscaling.monitored_nodes_labels": json.dumps(
-            app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_NODE_LABELS
+        AWSTagKey("io.simcore.autoscaling.version"): AWSTagValue(f"{VERSION}"),
+        AWSTagKey("io.simcore.autoscaling.monitored_nodes_labels"): AWSTagValue(
+            json.dumps(
+                app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_NODE_LABELS
+            )
         ),
-        "io.simcore.autoscaling.monitored_services_labels": json.dumps(
-            app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_SERVICE_LABELS
+        AWSTagKey("io.simcore.autoscaling.monitored_services_labels"): AWSTagValue(
+            json.dumps(
+                app_settings.AUTOSCALING_NODES_MONITORING.NODES_MONITORING_SERVICE_LABELS
+            )
         ),
         # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
-        "Name": f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}",
+        AWSTagKey("Name"): AWSTagValue(
+            f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}"
+        ),
     }
 
 
-def get_ec2_tags_computational(app_settings: ApplicationSettings) -> dict[str, str]:
+def get_ec2_tags_computational(app_settings: ApplicationSettings) -> EC2Tags:
     assert app_settings.AUTOSCALING_DASK  # nosec
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
     return {
-        "io.simcore.autoscaling.version": f"{VERSION}",
-        "io.simcore.autoscaling.dask-scheduler_url": json.dumps(
-            app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL
+        AWSTagKey("io.simcore.autoscaling.version"): AWSTagValue(f"{VERSION}"),
+        AWSTagKey("io.simcore.autoscaling.dask-scheduler_url"): AWSTagValue(
+            json.dumps(app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL)
         ),
         # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
-        "Name": f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}",
+        AWSTagKey("Name"): AWSTagValue(
+            f"{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_NAME_PREFIX}-{app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME}"
+        ),
     }
 
 
diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
index 5774f42e0e4..2415dbdb993 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
@@ -1,17 +1,18 @@
 from textwrap import dedent
 from typing import Final
 
-from aws_library.ec2.models import EC2Tags
+from aws_library.ec2.models import AWSTagKey, AWSTagValue, EC2Tags
 from models_library.users import UserID
 from models_library.wallets import WalletID
+from pydantic import parse_obj_as
 
 from .._meta import VERSION
 from ..core.settings import ApplicationSettings
 
 _APPLICATION_TAG_KEY: Final[str] = "io.simcore.clusters-keeper"
-_APPLICATION_VERSION_TAG: Final[EC2Tags] = {
-    f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"
-}
+_APPLICATION_VERSION_TAG: Final[EC2Tags] = parse_obj_as(
+    EC2Tags, {f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"}
+)
 
 HEARTBEAT_TAG_KEY: Final[str] = "last_heartbeat"
 CLUSTER_NAME_PREFIX: Final[str] = "osparc-computational-cluster-"
@@ -28,7 +29,11 @@ def get_cluster_name(
 
 
 def _minimal_identification_tag(app_settings: ApplicationSettings) -> EC2Tags:
-    return {".".join([_APPLICATION_TAG_KEY, "deploy"]): app_settings.SWARM_STACK_NAME}
+    return {
+        AWSTagKey(".".join([_APPLICATION_TAG_KEY, "deploy"])): AWSTagValue(
+            app_settings.SWARM_STACK_NAME
+        )
+    }
 
 
 def creation_ec2_tags(
@@ -40,11 +45,13 @@ def creation_ec2_tags(
         | _APPLICATION_VERSION_TAG
         | {
             # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
-            "Name": get_cluster_name(
-                app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=True
+            AWSTagKey("Name"): AWSTagValue(
+                get_cluster_name(
+                    app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=True
+                )
             ),
-            "user_id": f"{user_id}",
-            "wallet_id": f"{wallet_id}",
+            AWSTagKey("user_id"): AWSTagValue(f"{user_id}"),
+            AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}"),
         }
     )
 
@@ -58,8 +65,8 @@ def ec2_instances_for_user_wallet_filter(
 ) -> EC2Tags:
     return (
         _minimal_identification_tag(app_settings)
-        | {"user_id": f"{user_id}"}
-        | {"wallet_id": f"{wallet_id}"}
+        | {AWSTagKey("user_id"): AWSTagValue(f"{user_id}")}
+        | {AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}")}
     )
 
 

From ee3a65314e158492988ea590c003b1ac77541828 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 16:36:30 +0100
Subject: [PATCH 04/13] use correct type

---
 .../src/simcore_service_clusters_keeper/core/settings.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
index 34337e8dd4e..04c62f9c22d 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
@@ -2,7 +2,7 @@
 from functools import cached_property
 from typing import Any, ClassVar, Final, cast
 
-from aws_library.ec2.models import EC2InstanceBootSpecific
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2Tags
 from fastapi import FastAPI
 from models_library.basic_types import (
     BootModeEnum,
@@ -129,7 +129,7 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings):
         " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-key-pairs.html),"
         "this is required to start a new EC2 instance",
     )
-    PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: dict[str, str] = Field(
+    PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
         ...,
         description="Allows to define tags that should be added to the created EC2 instance default tags. "
         "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",

From bbdf85b0200fcdbc7ec738991a5f889a0bb1fb0f Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 16:58:52 +0100
Subject: [PATCH 05/13] now custom tag comes in

---
 .../aws-library/src/aws_library/ec2/models.py |  2 +-
 .../clusters-keeper/tests/unit/conftest.py    |  3 ++
 .../tests/unit/test_core_settings.py          | 45 +++++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py
index 26ec502d70b..ad5420a89ee 100644
--- a/packages/aws-library/src/aws_library/ec2/models.py
+++ b/packages/aws-library/src/aws_library/ec2/models.py
@@ -65,7 +65,7 @@ class EC2InstanceType:
 
 
 class AWSTagKey(ConstrainedStr):
-    regex = re.compile(r"^(?!.*(\.{1,2}|_index))[a-zA-Z0-9\+\-=\._:@]{1,128}$")
+    regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]{1,128}$")
 
 
 class AWSTagValue(ConstrainedStr):
diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py
index c664094f59a..735803b7439 100644
--- a/services/clusters-keeper/tests/unit/conftest.py
+++ b/services/clusters-keeper/tests/unit/conftest.py
@@ -113,6 +113,9 @@ def app_environment(
                     ]  # NOTE: we use example with custom script
                 }
             ),
+            "PRIMARY_EC2_INSTANCES_CUSTOM_TAGS": json.dumps(
+                {"osparc-tag": "clusters-keeper-machine"}
+            ),
             "CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES": "{}",
             "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps(
                 {
diff --git a/services/clusters-keeper/tests/unit/test_core_settings.py b/services/clusters-keeper/tests/unit/test_core_settings.py
index 895fc3618b8..4e1df0c9e06 100644
--- a/services/clusters-keeper/tests/unit/test_core_settings.py
+++ b/services/clusters-keeper/tests/unit/test_core_settings.py
@@ -54,3 +54,48 @@ def test_multiple_primary_ec2_instances_raises(
     )
     with pytest.raises(ValidationError, match="Only one exact value"):
         ApplicationSettings.create_from_envs()
+
+
+@pytest.mark.parametrize(
+    "invalid_tag",
+    [
+        {".": "single dot is invalid"},
+        {"..": "single 2 dots is invalid"},
+        {"": "empty tag key"},
+        {"/": "slash is invalid"},
+        {" ": "space is invalid"},
+    ],
+    ids=str,
+)
+def test_invalid_primary_custom_tags_raises(
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    invalid_tag: dict[str, str],
+):
+    setenvs_from_dict(
+        monkeypatch,
+        {"PRIMARY_EC2_INSTANCES_CUSTOM_TAGS": json.dumps(invalid_tag)},
+    )
+    with pytest.raises(ValidationError, match="string does not match regex"):
+        ApplicationSettings.create_from_envs()
+
+
+@pytest.mark.parametrize(
+    "valid_tag",
+    [
+        {"...": "3 dots is valid"},
+        {"..fdkjdlk..dsflkjsd=-lkjfie@": ""},
+        {"abcdef-lsaj+-=._:@": "values are able to take almost anything"},
+    ],
+    ids=str,
+)
+def test_valid_primary_custom_tags(
+    app_environment: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    valid_tag: dict[str, str],
+):
+    setenvs_from_dict(
+        monkeypatch,
+        {"PRIMARY_EC2_INSTANCES_CUSTOM_TAGS": json.dumps(valid_tag)},
+    )
+    ApplicationSettings.create_from_envs()

From d11e58154cf39b4f0378dcae691adcea20871ca2 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:04:25 +0100
Subject: [PATCH 06/13] pass the tags in

---
 .../src/simcore_service_clusters_keeper/utils/ec2.py             | 1 +
 1 file changed, 1 insertion(+)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
index 2415dbdb993..c7dab3e615d 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py
@@ -53,6 +53,7 @@ def creation_ec2_tags(
             AWSTagKey("user_id"): AWSTagValue(f"{user_id}"),
             AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}"),
         }
+        | app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_CUSTOM_TAGS
     )
 
 

From 0aebe38a0ddc96656dd01a932b364e656a53f65d Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:18:39 +0100
Subject: [PATCH 07/13] primary custom tags works

---
 services/clusters-keeper/tests/unit/conftest.py              | 2 +-
 services/clusters-keeper/tests/unit/test_modules_clusters.py | 5 +++--
 services/clusters-keeper/tests/unit/test_utils_ec2.py        | 1 +
 services/docker-compose.yml                                  | 1 +
 4 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py
index 735803b7439..8647b974f1f 100644
--- a/services/clusters-keeper/tests/unit/conftest.py
+++ b/services/clusters-keeper/tests/unit/conftest.py
@@ -114,7 +114,7 @@ def app_environment(
                 }
             ),
             "PRIMARY_EC2_INSTANCES_CUSTOM_TAGS": json.dumps(
-                {"osparc-tag": "clusters-keeper-machine"}
+                {"osparc-tag": "the pytest tag is here"}
             ),
             "CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES": "{}",
             "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps(
diff --git a/services/clusters-keeper/tests/unit/test_modules_clusters.py b/services/clusters-keeper/tests/unit/test_modules_clusters.py
index 6d0e82cd8da..5ce6672b0a3 100644
--- a/services/clusters-keeper/tests/unit/test_modules_clusters.py
+++ b/services/clusters-keeper/tests/unit/test_modules_clusters.py
@@ -71,7 +71,7 @@ async def _assert_cluster_instance_created(
     assert len(instances["Reservations"][0]["Instances"]) == 1
     assert "Tags" in instances["Reservations"][0]["Instances"][0]
     instance_ec2_tags = instances["Reservations"][0]["Instances"][0]["Tags"]
-    assert len(instance_ec2_tags) == 5
+    assert len(instance_ec2_tags) == 6
     assert all("Key" in x for x in instance_ec2_tags)
     assert all("Value" in x for x in instance_ec2_tags)
 
@@ -81,6 +81,7 @@ async def _assert_cluster_instance_created(
         "Name": f"{CLUSTER_NAME_PREFIX}manager-{app_settings.SWARM_STACK_NAME}-user_id:{user_id}-wallet_id:{wallet_id}",
         "user_id": f"{user_id}",
         "wallet_id": f"{wallet_id}",
+        "osparc-tag": "the pytest tag is here",
     }
     for tag in instances["Reservations"][0]["Instances"][0]["Tags"]:
         assert "Key" in tag
@@ -213,7 +214,7 @@ async def _assert_cluster_heartbeat_on_instance(
     assert len(instances["Reservations"][0]["Instances"]) == 1
     assert "Tags" in instances["Reservations"][0]["Instances"][0]
     instance_tags = instances["Reservations"][0]["Instances"][0]["Tags"]
-    assert len(instance_tags) == 6
+    assert len(instance_tags) == 7
     assert all("Key" in x for x in instance_tags)
     list_of_heartbeats = list(
         filter(lambda x: x["Key"] == HEARTBEAT_TAG_KEY, instance_tags)  # type:ignore
diff --git a/services/clusters-keeper/tests/unit/test_utils_ec2.py b/services/clusters-keeper/tests/unit/test_utils_ec2.py
index 51ccbe9a7da..ac400497a15 100644
--- a/services/clusters-keeper/tests/unit/test_utils_ec2.py
+++ b/services/clusters-keeper/tests/unit/test_utils_ec2.py
@@ -70,6 +70,7 @@ def test_creation_ec2_tags(
         "Name",
         "user_id",
         "wallet_id",
+        "osparc-tag",
     ]
     assert all(
         tag_key_name in received_tags for tag_key_name in EXPECTED_TAG_KEY_NAMES
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index eb401ae47ee..df66b973da4 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -157,6 +157,7 @@ services:
       - PRIMARY_EC2_INSTANCES_MAX_INSTANCES=${PRIMARY_EC2_INSTANCES_MAX_INSTANCES}
       - PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS=${PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS}
       - PRIMARY_EC2_INSTANCES_SUBNET_ID=${PRIMARY_EC2_INSTANCES_SUBNET_ID}
+      - PRIMARY_EC2_INSTANCES_CUSTOM_TAGS=${PRIMARY_EC2_INSTANCES_CUSTOM_TAGS}
       - RABBIT_HOST=${RABBIT_HOST}
       - RABBIT_PASSWORD=${RABBIT_PASSWORD}
       - RABBIT_PORT=${RABBIT_PORT}

From 9f263b1d2700a53b5eb64124735eb51bcdadc8e1 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:22:00 +0100
Subject: [PATCH 08/13] adding worker custom tags

---
 .../src/simcore_service_clusters_keeper/core/settings.py    | 6 ++++++
 services/clusters-keeper/tests/unit/conftest.py             | 3 +++
 services/docker-compose.yml                                 | 1 +
 3 files changed, 10 insertions(+)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
index 04c62f9c22d..1d9eaabb0e7 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py
@@ -85,6 +85,12 @@ class WorkersEC2InstancesSettings(BaseCustomSettings):
         "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
 
+    WORKERS_EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
+
     @validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES")
     @classmethod
     def check_valid_instance_names(
diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py
index 8647b974f1f..8d657b99689 100644
--- a/services/clusters-keeper/tests/unit/conftest.py
+++ b/services/clusters-keeper/tests/unit/conftest.py
@@ -130,6 +130,9 @@ def app_environment(
             ),
             "WORKERS_EC2_INSTANCES_SUBNET_ID": faker.pystr(),
             "WORKERS_EC2_INSTANCES_KEY_NAME": faker.pystr(),
+            "WORKERS_EC2_INSTANCES_CUSTOM_TAGS": json.dumps(
+                {"osparc-tag": "the pytest worker tag value is here"}
+            ),
         },
     )
     return mock_env_devel_environment | envs
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index df66b973da4..08da1b8ea93 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -173,6 +173,7 @@ services:
       - WORKERS_EC2_INSTANCES_MAX_INSTANCES=${WORKERS_EC2_INSTANCES_MAX_INSTANCES}
       - WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS=${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS}
       - WORKERS_EC2_INSTANCES_SUBNET_ID=${WORKERS_EC2_INSTANCES_SUBNET_ID}
+      - WORKERS_EC2_INSTANCES_CUSTOM_TAGS=${WORKERS_EC2_INSTANCES_CUSTOM_TAGS}
 
   director:
     image: ${DOCKER_REGISTRY:-itisfoundation}/director:${DOCKER_IMAGE_TAG:-latest}

From cec1266d75db753d257b15b845964d325cd00e4d Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:24:36 +0100
Subject: [PATCH 09/13] pass the worker tags to the autoscaling

---
 .../src/simcore_service_clusters_keeper/data/docker-compose.yml  | 1 +
 .../src/simcore_service_clusters_keeper/utils/clusters.py        | 1 +
 services/clusters-keeper/tests/manual/README.md                  | 1 +
 3 files changed, 3 insertions(+)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml
index bb4e22973ee..9c4454ea348 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml
@@ -61,6 +61,7 @@ services:
       EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS}
       EC2_INSTANCES_SUBNET_ID: ${WORKERS_EC2_INSTANCES_SUBNET_ID}
       EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}
+      EC2_INSTANCES_CUSTOM_TAGS: ${WORKERS_EC2_INSTANCES_CUSTOM_TAGS}
       LOG_FORMAT_LOCAL_DEV_ENABLED: 1
       LOG_LEVEL: ${LOG_LEVEL:-WARNING}
       REDIS_HOST: redis
diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
index 9db8cbbea79..5ecb2c656e5 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
@@ -58,6 +58,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
+        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS}",
         f"LOG_LEVEL={app_settings.LOG_LEVEL}",
     ]
 
diff --git a/services/clusters-keeper/tests/manual/README.md b/services/clusters-keeper/tests/manual/README.md
index 08aad73269d..6399f4664c5 100644
--- a/services/clusters-keeper/tests/manual/README.md
+++ b/services/clusters-keeper/tests/manual/README.md
@@ -79,6 +79,7 @@ WORKERS_EC2_INSTANCES_MAX_INSTANCES=10
 WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS="[\"XXXXXXX\"]"
 WORKERS_EC2_INSTANCES_SUBNET_ID=XXXXXXX
 WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION="00:03:00"
+WORKERS_EC2_INSTANCES_CUSTOM_TAGS='{"osparc-tag": "some fun tag value"}'
 ```
 
 5. start osparc

From 3b1333ebd26458795681ec260561705a8f126181 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:50:58 +0100
Subject: [PATCH 10/13] autoscaling now allows to setup custom tags

---
 packages/aws-library/src/aws_library/ec2/models.py         | 5 ++++-
 .../src/simcore_service_autoscaling/core/settings.py       | 7 ++++++-
 .../modules/auto_scaling_core.py                           | 5 ++++-
 .../src/simcore_service_autoscaling/utils/utils_ec2.py     | 2 +-
 services/autoscaling/tests/unit/conftest.py                | 7 +++++++
 .../tests/unit/test_modules_auto_scaling_computational.py  | 3 +++
 .../tests/unit/test_modules_auto_scaling_dynamic.py        | 3 +++
 services/docker-compose.yml                                | 1 +
 8 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/packages/aws-library/src/aws_library/ec2/models.py b/packages/aws-library/src/aws_library/ec2/models.py
index ad5420a89ee..529d1ba8d09 100644
--- a/packages/aws-library/src/aws_library/ec2/models.py
+++ b/packages/aws-library/src/aws_library/ec2/models.py
@@ -65,11 +65,14 @@ class EC2InstanceType:
 
 
 class AWSTagKey(ConstrainedStr):
+    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
     regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]{1,128}$")
 
 
 class AWSTagValue(ConstrainedStr):
-    regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\._:/@]{0,256}$")
+    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
+    # quotes []{} were added as it allows to json encode. it seems to be accepted as a value
+    regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\._:/@\"\'\[\]\{\}]{0,256}$")
 
 
 EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue]
diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py
index 12ce642d348..b04cd7ab076 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py
@@ -2,7 +2,7 @@
 from functools import cached_property
 from typing import Any, ClassVar, Final, cast
 
-from aws_library.ec2.models import EC2InstanceBootSpecific
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2Tags
 from fastapi import FastAPI
 from models_library.basic_types import (
     BootModeEnum,
@@ -101,6 +101,11 @@ class EC2InstancesSettings(BaseCustomSettings):
         description="Time after which an EC2 instance may be terminated (0<=T<=59 minutes, is automatically capped)"
         "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formating)",
     )
+    EC2_INSTANCES_CUSTOM_TAGS: EC2Tags = Field(
+        ...,
+        description="Allows to define tags that should be added to the created EC2 instance default tags. "
+        "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]",
+    )
 
     @validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION")
     @classmethod
diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py
index 208f10f5790..d9ded9d9150 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py
@@ -518,7 +518,10 @@ async def _start_instances(
     ec2_client = get_ec2_client(app)
     app_settings = get_application_settings(app)
     assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
-    new_instance_tags = auto_scaling_mode.get_ec2_tags(app)
+    new_instance_tags = (
+        auto_scaling_mode.get_ec2_tags(app)
+        | app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_CUSTOM_TAGS
+    )
     capped_needed_machines = {}
     try:
         capped_needed_machines = await _cap_needed_instances(
diff --git a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
index e167a36fd9c..afb4d224311 100644
--- a/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
+++ b/services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py
@@ -51,7 +51,7 @@ def get_ec2_tags_computational(app_settings: ApplicationSettings) -> EC2Tags:
     return {
         AWSTagKey("io.simcore.autoscaling.version"): AWSTagValue(f"{VERSION}"),
         AWSTagKey("io.simcore.autoscaling.dask-scheduler_url"): AWSTagValue(
-            json.dumps(app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL)
+            f"{app_settings.AUTOSCALING_DASK.DASK_MONITORING_URL}"
         ),
         # NOTE: this one gets special treatment in AWS GUI and is applied to the name of the instance
         AWSTagKey("Name"): AWSTagValue(
diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py
index c6a1dbe8191..c13c9be2f16 100644
--- a/services/autoscaling/tests/unit/conftest.py
+++ b/services/autoscaling/tests/unit/conftest.py
@@ -132,6 +132,13 @@ def app_environment(
                     for ec2_type_name in ec2_instances
                 }
             ),
+            "EC2_INSTANCES_CUSTOM_TAGS": json.dumps(
+                {
+                    "user_id": "32",
+                    "wallet_id": "3245",
+                    "osparc-tag": "some whatever value",
+                }
+            ),
         },
     )
     return mock_env_devel_environment | envs
diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py
index f204e659cf3..f909caab521 100644
--- a/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py
+++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py
@@ -115,6 +115,9 @@ async def _assert_ec2_instances(
                 "io.simcore.autoscaling.version",
                 "io.simcore.autoscaling.dask-scheduler_url",
                 "Name",
+                "user_id",
+                "wallet_id",
+                "osparc-tag",
             ]
             for tag_dict in instance["Tags"]:
                 assert "Key" in tag_dict
diff --git a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py
index fbabf7f3801..ef1d31de6f8 100644
--- a/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py
+++ b/services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py
@@ -381,6 +381,9 @@ async def _assert_ec2_instances(
                 "io.simcore.autoscaling.monitored_nodes_labels",
                 "io.simcore.autoscaling.monitored_services_labels",
                 "Name",
+                "user_id",
+                "wallet_id",
+                "osparc-tag",
             ]
             for tag_dict in instance["Tags"]:
                 assert "Key" in tag_dict
diff --git a/services/docker-compose.yml b/services/docker-compose.yml
index 08da1b8ea93..be07ac05539 100644
--- a/services/docker-compose.yml
+++ b/services/docker-compose.yml
@@ -79,6 +79,7 @@ services:
       - EC2_INSTANCES_SECURITY_GROUP_IDS=${EC2_INSTANCES_SECURITY_GROUP_IDS}
       - EC2_INSTANCES_SUBNET_ID=${EC2_INSTANCES_SUBNET_ID}
       - EC2_INSTANCES_KEY_NAME=${EC2_INSTANCES_KEY_NAME}
+      - EC2_INSTANCES_CUSTOM_TAGS=${EC2_INSTANCES_CUSTOM_TAGS}
 
       - AUTOSCALING_NODES_MONITORING=${AUTOSCALING_NODES_MONITORING} # dyn autoscaling
       - NODES_MONITORING_NODE_LABELS=${NODES_MONITORING_NODE_LABELS}

From 5ae5e138b314590946ad01888f51219ed1059926 Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 17:55:32 +0100
Subject: [PATCH 11/13] user_id and wallet_id are passed to autoscsaling custom
 tags

---
 .../modules/clusters.py                                | 10 ++++++++--
 .../simcore_service_clusters_keeper/utils/clusters.py  |  6 ++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py
index cac174fd698..c4171cdc037 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters.py
@@ -3,6 +3,8 @@
 
 from aws_library.ec2.client import SimcoreEC2API
 from aws_library.ec2.models import (
+    AWSTagKey,
+    AWSTagValue,
     EC2InstanceBootSpecific,
     EC2InstanceConfig,
     EC2InstanceData,
@@ -69,10 +71,14 @@ async def create_cluster(
         tags=creation_ec2_tags(app_settings, user_id=user_id, wallet_id=wallet_id),
         startup_script=create_startup_script(
             app_settings,
-            get_cluster_name(
+            cluster_machines_name_prefix=get_cluster_name(
                 app_settings, user_id=user_id, wallet_id=wallet_id, is_manager=False
             ),
-            ec2_instance_boot_specs,
+            ec2_boot_specific=ec2_instance_boot_specs,
+            additional_custom_tags={
+                AWSTagKey("user_id"): AWSTagValue(f"{user_id}"),
+                AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}"),
+            },
         ),
         ami_id=ec2_instance_boot_specs.ami_id,
         key_name=app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_KEY_NAME,
diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
index 5ecb2c656e5..85e5361bf97 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
@@ -4,7 +4,7 @@
 import json
 from typing import Any, Final
 
-from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData
+from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData, EC2Tags
 from fastapi.encoders import jsonable_encoder
 from models_library.api_schemas_clusters_keeper.clusters import (
     ClusterState,
@@ -32,8 +32,10 @@ def _docker_compose_yml_base64_encoded() -> str:
 
 def create_startup_script(
     app_settings: ApplicationSettings,
+    *,
     cluster_machines_name_prefix: str,
     ec2_boot_specific: EC2InstanceBootSpecific,
+    additional_custom_tags: EC2Tags,
 ) -> str:
     assert app_settings.CLUSTERS_KEEPER_EC2_ACCESS  # nosec
     assert app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES  # nosec
@@ -58,7 +60,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
-        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS}",
+        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags}",
         f"LOG_LEVEL={app_settings.LOG_LEVEL}",
     ]
 

From 53f68bd72d488216ee0892c15fdbf5ef8aa05ddb Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Thu, 7 Dec 2023 18:21:08 +0100
Subject: [PATCH 12/13] ensure dict are correctly setup

---
 .../utils/clusters.py                         |  2 +-
 .../tests/unit/test_utils_clusters.py         | 31 +++++++++++++++++--
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
index 85e5361bf97..341d7f9d749 100644
--- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
+++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py
@@ -60,7 +60,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}",
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
-        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags}",
+        f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}",
         f"LOG_LEVEL={app_settings.LOG_LEVEL}",
     ]
 
diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py
index 9c56d43e34a..cbb188a0c16 100644
--- a/services/clusters-keeper/tests/unit/test_utils_clusters.py
+++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py
@@ -7,7 +7,12 @@
 from typing import Any
 
 import pytest
-from aws_library.ec2.models import EC2InstanceBootSpecific, EC2InstanceData
+from aws_library.ec2.models import (
+    AWSTagKey,
+    AWSTagValue,
+    EC2InstanceBootSpecific,
+    EC2InstanceData,
+)
 from faker import Faker
 from models_library.api_schemas_clusters_keeper.clusters import ClusterState
 from pytest_simcore.helpers.utils_envs import EnvVarsDict
@@ -45,8 +50,14 @@ def test_create_startup_script(
     clusters_keeper_docker_compose: dict[str, Any],
     ec2_boot_specs: EC2InstanceBootSpecific,
 ):
+    additional_custom_tags = {
+        AWSTagKey("pytest-tag-key"): AWSTagValue("pytest-tag-value")
+    }
     startup_script = create_startup_script(
-        app_settings, cluster_machines_name_prefix, ec2_boot_specs
+        app_settings,
+        cluster_machines_name_prefix=cluster_machines_name_prefix,
+        ec2_boot_specific=ec2_boot_specs,
+        additional_custom_tags=additional_custom_tags,
     )
     assert isinstance(startup_script, str)
     assert len(ec2_boot_specs.custom_boot_scripts) > 0
@@ -104,6 +115,22 @@ def test_create_startup_script(
         re.search(rf"{i}=\[(\\\".+\\\")*\]", startup_script) for i in list_settings
     )
 
+    # check dicts have \' in front
+    dict_settings = [
+        "WORKERS_EC2_INSTANCES_ALLOWED_TYPES",
+        "WORKERS_EC2_INSTANCES_CUSTOM_TAGS",
+    ]
+    assert all(
+        re.search(rf"{i}=\'{{(\".+\":\s\".*\")+}}\'", startup_script)
+        for i in dict_settings
+    )
+
+    # check the additional tags are in
+    assert all(
+        f'"{key}": "{value}"' in startup_script
+        for key, value in additional_custom_tags.items()
+    )
+
 
 @pytest.mark.parametrize(
     "ec2_state, expected_cluster_state",

From cded45f55d2617a262868954349c8d7512635d7f Mon Sep 17 00:00:00 2001
From: sanderegg <35365065+sanderegg@users.noreply.github.com>
Date: Mon, 11 Dec 2023 08:35:17 +0100
Subject: [PATCH 13/13] fix pylint error

---
 .pylintrc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pylintrc b/.pylintrc
index f3f789e9c63..b680ebc6541 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -92,7 +92,7 @@ persistent=yes
 py-version=3.10
 
 # Discover python modules and packages in the file system subtree.
-recursive=no
+recursive=true
 
 # When enabled, pylint would attempt to guess common misconfiguration and emit
 # user-friendly hints instead of false-positive error messages.