gRPC load balancing via Ambassador

cluster-manager/src/main/java/io/seldon/clustermanager/k8s/Constants.java

@@ -26,4 +26,5 @@ public class Constants {
     public static final String ENGINE_SEPARATE_ANNOTATION = "seldon.io/engine-separate-pod";
     public static final String REST_READ_TIMEOUT_ANNOTATION = "seldon.io/rest-read-timeout";
     public static final String GRPC_READ_TIMEOUT_ANNOTATION = "seldon.io/grpc-read-timeout";
+    public static final String HEADLESS_SVC_ANNOTATION = "seldon.io/headless-svc";
 }

cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java

@@ -382,6 +382,7 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) {
 		SeldonDeployment.Builder mlBuilder = SeldonDeployment.newBuilder(mlDep);
 		String deploymentName = mlDep.getMetadata().getName();
 		final boolean separateEnginePod = SeldonDeploymentUtils.hasSeparateEnginePodAnnotation(mlDep);
+
 		final String namespace = (StringUtils.isEmpty(mlDep.getMetadata().getNamespace())) ? "default" : mlDep.getMetadata().getNamespace();
 
 		for(int pbIdx=0;pbIdx<mlDep.getSpec().getPredictorsCount();pbIdx++)
@@ -810,7 +811,8 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon
 			}
 		}
 
-		Service s = Service.newBuilder()
+		final boolean headlessSvc = SeldonDeploymentUtils.hasHeadlessSvcAnnotation(mlDep);
+		Service.Builder svcBuilder = Service.newBuilder()
 					.setMetadata(ObjectMeta.newBuilder()
 							.setName(serviceLabel)
 							.putLabels(LABEL_SELDON_APP, serviceLabel)
@@ -831,12 +833,15 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon
                                     .setTargetPort(IntOrString.newBuilder().setIntVal(clusterManagerProperites.getEngineGrpcContainerPort()))
                                     .setName("grpc")
                                     )
-							.setType("ClusterIP")
 							.putSelector(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP,serviceLabel)
-							)
-				.build();
-
-		services.add(s);
+							);
+		if (headlessSvc)
+		{
+			logger.info("Creating headless service for ",mlDep.getSpec().getName());
+			svcBuilder.getSpecBuilder().setClusterIP("None");
+		}
+
+		services.add(svcBuilder.build());
 
 
 

cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentUtils.java

@@ -72,4 +72,9 @@ public static boolean hasSeparateEnginePodAnnotation(SeldonDeployment mlDep)
 		return Boolean.parseBoolean(mlDep.getSpec().getAnnotationsOrDefault(Constants.ENGINE_SEPARATE_ANNOTATION, "false"));
 	}
 
+	public static boolean hasHeadlessSvcAnnotation(SeldonDeployment mlDep)
+	{
+		return Boolean.parseBoolean(mlDep.getSpec().getAnnotationsOrDefault(Constants.HEADLESS_SVC_ANNOTATION, "false"));
+	}
+
 }

docs/annotations.md

@@ -31,6 +31,9 @@ You can configure aspects of Seldon Core via annotations in the SeldonDeployment
   * ```seldon.io/engine-separate-pod``` : Use a separate pod for the service orchestrator
     * Locations : SeldonDeployment.spec.annotations
     * [Example](../notebooks/resources/model_svcorch_sep.json)
+  * ```seldon.io/headless-svc``` : Run main endpoint as headless kubernetes service. This is required for gRPC load balancing via Ambassador.
+    * Locations : SeldonDeployment.spec.annotations
+    * [Example](../notebooks/resources/grpc_load_balancing_ambassador.json)
 
 ## API OAuth Gateway Annotations
 The API OAuth Gateway, if used, can also have the following annotations:

notebooks/resources/grpc_load_balancing_ambassador.json

@@ -0,0 +1,52 @@
+{
+    "apiVersion": "machinelearning.seldon.io/v1alpha2",
+    "kind": "SeldonDeployment",
+    "metadata": {
+        "labels": {
+            "app": "seldon"
+        },
+        "name": "seldon-model"
+    },
+    "spec": {
+	"annotations": {
+	    "seldon.io/headless-svc":"true"
+	},
+        "name": "test-deployment",
+        "oauth_key": "oauth-key",
+        "oauth_secret": "oauth-secret",
+        "predictors": [
+            {
+                "componentSpecs": [{
+                    "spec": {
+                        "containers": [
+                            {
+                                "image": "seldonio/mock_classifier_grpc:1.0",
+                                "imagePullPolicy": "IfNotPresent",
+                                "name": "classifier",
+                                "resources": {
+                                    "requests": {
+                                        "memory": "1Mi"
+                                    }
+                                }
+                            }
+                        ],
+                        "terminationGracePeriodSeconds": 1
+                    }
+                }],
+                "graph": {
+                    "children": [],
+                    "name": "classifier",
+                    "endpoint": {
+			"type" : "GRPC"
+		    },
+                    "type": "MODEL"
+                },
+                "name": "example",
+                "replicas": 4,
+		"labels": {
+		    "version" : "v1"
+		}
+            }
+        ]
+    }
+}