Sync branch

.github/workflows/codebuild-ci.yml

@@ -55,7 +55,7 @@ jobs:
       - name: Run Codestyle & Doc Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-codestyle-doc-tests
+          project-name: ${{ github.event.repository.name }}-ci-codestyle-doc-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
   unit-tests:
     runs-on: ubuntu-latest
@@ -74,7 +74,7 @@ jobs:
       - name: Run Unit Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-unit-tests
+          project-name: ${{ github.event.repository.name }}-ci-unit-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'
           env-vars-for-codebuild: |
             PY_VERSION
@@ -93,5 +93,5 @@ jobs:
       - name: Run Integ Tests
         uses: aws-actions/aws-codebuild-run-build@v1
         with:
-          project-name: sagemaker-python-sdk-ci-integ-tests
+          project-name: ${{ github.event.repository.name }}-ci-integ-tests
           source-version-override: 'refs/pull/${{ github.event.pull_request.number }}/head^{${{ github.event.pull_request.head.sha }}}'

CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## v2.221.1 (2024-05-22)
+
+### Bug Fixes and Other Changes
+
+ * Convert pytorchddp distribution to smdistributed distribution
+ * Add tei cpu image
+
+## v2.221.0 (2024-05-20)
+
+### Features
+
+ * onboard tei image config to pysdk
+
+### Bug Fixes and Other Changes
+
+ * JS Model with non-TGI/non-DJL deployment failure
+ * cover tei with image_uris.retrieve API
+ * Add more debuging
+ * model builder limited container support for endpoint mode.
+ * Image URI should take precedence for HF models
+
 ## v2.220.0 (2024-05-15)
 
 ### Features

VERSION

@@ -1 +1 @@
-2.220.1.dev0
+2.221.2.dev0

requirements/extras/test_requirements.txt

@@ -12,7 +12,7 @@ awslogs==0.14.0
 black==24.3.0
 stopit==1.1.2
 # Update tox.ini to have correct version of airflow constraints file
-apache-airflow==2.9.0
+apache-airflow==2.9.1
 apache-airflow-providers-amazon==7.2.1
 attrs>=23.1.0,<24
 fabric==2.6.0

src/sagemaker/enums.py

@@ -28,3 +28,15 @@ class EndpointType(Enum):
     INFERENCE_COMPONENT_BASED = (
         "InferenceComponentBased"  # Amazon SageMaker Inference Component Based Endpoint
     )
+
+
+class RoutingStrategy(Enum):
+    """Strategy for routing https traffics."""
+
+    RANDOM = "RANDOM"
+    """The endpoint routes each request to a randomly chosen instance.
+    """
+    LEAST_OUTSTANDING_REQUESTS = "LEAST_OUTSTANDING_REQUESTS"
+    """The endpoint routes requests to the specific instances that have
+    more capacity to process them.
+    """

src/sagemaker/fw_utils.py

@@ -145,22 +145,6 @@
     ],
 }
 
-PYTORCHDDP_SUPPORTED_FRAMEWORK_VERSIONS = [
-    "1.10",
-    "1.10.0",
-    "1.10.2",
-    "1.11",
-    "1.11.0",
-    "1.12",
-    "1.12.0",
-    "1.12.1",
-    "1.13.1",
-    "2.0.0",
-    "2.0.1",
-    "2.1.0",
-    "2.2.0",
-]
-
 TORCH_DISTRIBUTED_GPU_SUPPORTED_FRAMEWORK_VERSIONS = [
     "1.13.1",
     "2.0.0",
@@ -795,7 +779,6 @@ def _validate_smdataparallel_args(
 
     Raises:
         ValueError: if
-            (`instance_type` is not in SM_DATAPARALLEL_SUPPORTED_INSTANCE_TYPES or
             `py_version` is not python3 or
             `framework_version` is not in SM_DATAPARALLEL_SUPPORTED_FRAMEWORK_VERSION
     """
@@ -806,17 +789,10 @@ def _validate_smdataparallel_args(
     if not smdataparallel_enabled:
         return
 
-    is_instance_type_supported = instance_type in SM_DATAPARALLEL_SUPPORTED_INSTANCE_TYPES
-
     err_msg = ""
 
-    if not is_instance_type_supported:
-        # instance_type is required
-        err_msg += (
-            f"Provided instance_type {instance_type} is not supported by smdataparallel.\n"
-            "Please specify one of the supported instance types:"
-            f"{SM_DATAPARALLEL_SUPPORTED_INSTANCE_TYPES}\n"
-        )
+    if not instance_type:
+        err_msg += "Please specify an instance_type for smdataparallel.\n"
 
     if not image_uri:
         # ignore framework_version & py_version if image_uri is set
@@ -928,13 +904,6 @@ def validate_distribution(
             )
             if framework_name and framework_name == "pytorch":
                 # We need to validate only for PyTorch framework
-                validate_pytorch_distribution(
-                    distribution=validated_distribution,
-                    framework_name=framework_name,
-                    framework_version=framework_version,
-                    py_version=py_version,
-                    image_uri=image_uri,
-                )
                 validate_torch_distributed_distribution(
                     instance_type=instance_type,
                     distribution=validated_distribution,
@@ -968,13 +937,6 @@ def validate_distribution(
         )
         if framework_name and framework_name == "pytorch":
             # We need to validate only for PyTorch framework
-            validate_pytorch_distribution(
-                distribution=validated_distribution,
-                framework_name=framework_name,
-                framework_version=framework_version,
-                py_version=py_version,
-                image_uri=image_uri,
-            )
             validate_torch_distributed_distribution(
                 instance_type=instance_type,
                 distribution=validated_distribution,
@@ -1023,63 +985,6 @@ def validate_distribution_for_instance_type(instance_type, distribution):
         raise ValueError(err_msg)
 
 
-def validate_pytorch_distribution(
-    distribution, framework_name, framework_version, py_version, image_uri
-):
-    """Check if pytorch distribution strategy is correctly invoked by the user.
-
-    Args:
-        distribution (dict): A dictionary with information to enable distributed training.
-            (Defaults to None if distributed training is not enabled.) For example:
-
-            .. code:: python
-
-                {
-                    "pytorchddp": {
-                        "enabled": True
-                    }
-                }
-        framework_name (str): A string representing the name of framework selected.
-        framework_version (str): A string representing the framework version selected.
-        py_version (str): A string representing the python version selected.
-        image_uri (str): A string representing a Docker image URI.
-
-    Raises:
-        ValueError: if
-            `py_version` is not python3 or
-            `framework_version` is not in PYTORCHDDP_SUPPORTED_FRAMEWORK_VERSIONS
-    """
-    if framework_name and framework_name != "pytorch":
-        # We need to validate only for PyTorch framework
-        return
-
-    pytorch_ddp_enabled = False
-    if "pytorchddp" in distribution:
-        pytorch_ddp_enabled = distribution.get("pytorchddp").get("enabled", False)
-    if not pytorch_ddp_enabled:
-        # Distribution strategy other than pytorchddp is selected
-        return
-
-    err_msg = ""
-    if not image_uri:
-        # ignore framework_version and py_version if image_uri is set
-        # in case image_uri is not set, then both are mandatory
-        if framework_version not in PYTORCHDDP_SUPPORTED_FRAMEWORK_VERSIONS:
-            err_msg += (
-                f"Provided framework_version {framework_version} is not supported by"
-                " pytorchddp.\n"
-                "Please specify one of the supported framework versions:"
-                f" {PYTORCHDDP_SUPPORTED_FRAMEWORK_VERSIONS} \n"
-            )
-        if "py3" not in py_version:
-            err_msg += (
-                f"Provided py_version {py_version} is not supported by pytorchddp.\n"
-                "Please specify py_version>=py3"
-            )
-    if err_msg:
-        raise ValueError(err_msg)
-
-
 def validate_torch_distributed_distribution(
     instance_type,
     distribution,

src/sagemaker/huggingface/llm_utils.py

@@ -65,6 +65,20 @@ def get_huggingface_llm_image_uri(
             image_scope="inference",
             inference_tool="neuronx",
         )
+    if backend == "huggingface-tei":
+        return image_uris.retrieve(
+            "huggingface-tei",
+            region=region,
+            version=version,
+            image_scope="inference",
+        )
+    if backend == "huggingface-tei-cpu":
+        return image_uris.retrieve(
+            "huggingface-tei-cpu",
+            region=region,
+            version=version,
+            image_scope="inference",
+        )
     if backend == "lmi":
         version = version or "0.24.0"
         return image_uris.retrieve(framework="djl-deepspeed", region=region, version=version)

src/sagemaker/huggingface/model.py

@@ -334,6 +334,7 @@ def deploy(
             endpoint_type=kwargs.get("endpoint_type", None),
             resources=kwargs.get("resources", None),
             managed_instance_scaling=kwargs.get("managed_instance_scaling", None),
+            routing_config=kwargs.get("routing_config", None),
         )
 
     def register(

src/sagemaker/image_uri_config/djl-lmi.json

@@ -0,0 +1,39 @@
+{
+    "scope": [
+        "inference"
+    ],
+    "versions": {
+        "0.28.0": {
+            "registries": {
+                "af-south-1": "626614931356",
+                "il-central-1": "780543022126",
+                "ap-east-1": "871362719292",
+                "ap-northeast-1": "763104351884",
+                "ap-northeast-2": "763104351884",
+                "ap-northeast-3": "364406365360",
+                "ap-south-1": "763104351884",
+                "ap-southeast-1": "763104351884",
+                "ap-southeast-2": "763104351884",
+                "ap-southeast-3": "907027046896",
+                "ca-central-1": "763104351884",
+                "cn-north-1": "727897471807",
+                "cn-northwest-1": "727897471807",
+                "eu-central-1": "763104351884",
+                "eu-north-1": "763104351884",
+                "eu-west-1": "763104351884",
+                "eu-west-2": "763104351884",
+                "eu-west-3": "763104351884",
+                "eu-south-1": "692866216735",
+                "me-south-1": "217643126080",
+                "sa-east-1": "763104351884",
+                "us-east-1": "763104351884",
+                "us-east-2": "763104351884",
+                "us-west-1": "763104351884",
+                "us-west-2": "763104351884",
+                "ca-west-1": "204538143572"
+            },
+            "repository": "djl-inference",
+            "tag_prefix": "0.28.0-lmi10.0.0-cu124"
+        }
+    }
+}

src/sagemaker/image_uri_config/djl-neuronx.json

@@ -3,6 +3,24 @@
         "inference"
     ],
     "versions": {
+        "0.28.0": {
+            "registries": {
+                "ap-northeast-1": "763104351884",
+                "ap-south-1": "763104351884",
+                "ap-southeast-1": "763104351884",
+                "ap-southeast-2": "763104351884",
+                "eu-central-1": "763104351884",
+                "eu-west-1": "763104351884",
+                "eu-west-3": "763104351884",
+                "sa-east-1": "763104351884",
+                "us-east-1": "763104351884",
+                "us-east-2": "763104351884",
+                "us-west-2": "763104351884",
+                "ca-west-1": "204538143572"
+            },
+            "repository": "djl-inference",
+            "tag_prefix": "0.28.0-neuronx-sdk2.18.2"
+        },
          "0.27.0": {
             "registries": {
                 "ap-northeast-1": "763104351884",

src/sagemaker/image_uri_config/djl-tensorrtllm.json

@@ -3,6 +3,38 @@
         "inference"
     ],
     "versions": {
+        "0.28.0": {
+            "registries": {
+                "af-south-1": "626614931356",
+                "il-central-1": "780543022126",
+                "ap-east-1": "871362719292",
+                "ap-northeast-1": "763104351884",
+                "ap-northeast-2": "763104351884",
+                "ap-northeast-3": "364406365360",
+                "ap-south-1": "763104351884",
+                "ap-southeast-1": "763104351884",
+                "ap-southeast-2": "763104351884",
+                "ap-southeast-3": "907027046896",
+                "ca-central-1": "763104351884",
+                "cn-north-1": "727897471807",
+                "cn-northwest-1": "727897471807",
+                "eu-central-1": "763104351884",
+                "eu-north-1": "763104351884",
+                "eu-west-1": "763104351884",
+                "eu-west-2": "763104351884",
+                "eu-west-3": "763104351884",
+                "eu-south-1": "692866216735",
+                "me-south-1": "217643126080",
+                "sa-east-1": "763104351884",
+                "us-east-1": "763104351884",
+                "us-east-2": "763104351884",
+                "us-west-1": "763104351884",
+                "us-west-2": "763104351884",
+                "ca-west-1": "204538143572"
+            },
+            "repository": "djl-inference",
+            "tag_prefix": "0.28.0-tensorrtllm0.9.0-cu122"
+        },
         "0.27.0": {
             "registries": {
                 "af-south-1": "626614931356",