From 17435e2d238471d099ad5cb7db154da0c982df21 Mon Sep 17 00:00:00 2001 From: ryandawsonuk Date: Thu, 4 Apr 2019 16:47:57 +0100 Subject: [PATCH 1/4] ambassador v1 api --- .../clustermanager/k8s/SeldonDeploymentOperatorImpl.java | 8 ++++---- .../seldon/clustermanager/k8s/AmbassadorConfigTest.java | 8 ++++---- doc/source/graph/ambassador.md | 2 +- doc/source/workflow/troubleshooting.md | 2 ++ examples/ambassador/custom/model_custom_ambassador.json | 2 +- 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index 4e85c1246b..97a1273dff 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -519,7 +519,7 @@ private String getAmbassadorAnnotation(SeldonDeployment mlDep,String serviceName final String customRegexHeader = mlDep.getSpec().getAnnotationsOrDefault(Constants.AMBASSADOR_REGEX_HEADER_ANNOTATION, null); final String restMapping = "---\n"+ - "apiVersion: ambassador/v0\n" + + "apiVersion: ambassador/v1\n" + "kind: Mapping\n" + "name: seldon_"+mlDep.getMetadata().getName()+"_rest_mapping\n" + "prefix: /seldon/"+serviceNameExternal+"/\n" + @@ -530,7 +530,7 @@ private String getAmbassadorAnnotation(SeldonDeployment mlDep,String serviceName (StringUtils.isNotEmpty(weight) ? ("weight: "+ weight + "\n") : "") + (StringUtils.isNotEmpty(shadowing) ? ("shadow: true\n") : ""); final String grpcMapping = "---\n"+ - "apiVersion: ambassador/v0\n" + + "apiVersion: ambassador/v1\n" + "kind: Mapping\n" + "name: "+mlDep.getMetadata().getName()+"_grpc_mapping\n" + "grpc: true\n" + @@ -546,7 +546,7 @@ private String getAmbassadorAnnotation(SeldonDeployment mlDep,String serviceName (StringUtils.isNotEmpty(shadowing) ? ("shadow: true\n") : ""); final String restMappingNamespaced = "---\n"+ - "apiVersion: ambassador/v0\n" + + "apiVersion: ambassador/v1\n" + "kind: Mapping\n" + "name: seldon_"+namespace+"_"+mlDep.getMetadata().getName()+"_rest_mapping\n" + "prefix: /seldon/"+namespace+"/"+serviceNameExternal+"/\n" + @@ -558,7 +558,7 @@ private String getAmbassadorAnnotation(SeldonDeployment mlDep,String serviceName (StringUtils.isNotEmpty(shadowing) ? ("shadow: true\n") : ""); final String grpcMappingNamespaced = "---\n"+ - "apiVersion: ambassador/v0\n" + + "apiVersion: ambassador/v1\n" + "kind: Mapping\n" + "name: "+namespace+"_"+mlDep.getMetadata().getName()+"_grpc_mapping\n" + "grpc: true\n" + diff --git a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/AmbassadorConfigTest.java b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/AmbassadorConfigTest.java index 69a69a78f8..7ed6d8b67b 100644 --- a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/AmbassadorConfigTest.java +++ b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/AmbassadorConfigTest.java @@ -43,7 +43,7 @@ public void checkAmbassadorCanary() throws IOException, SeldonDeploymentExceptio System.out.println(ambassadorConfig); Assert.assertTrue(ambassadorConfig.indexOf("weight: 25\n")>0); Assert.assertTrue(ambassadorConfig.indexOf("prefix: /seldon/default/example/\n")>0); - Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v0")==4); + Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v1")==4); } @Test @@ -61,7 +61,7 @@ public void checkAmbassadorShadow() throws IOException, SeldonDeploymentExceptio System.out.println(ambassadorConfig); Assert.assertTrue(ambassadorConfig.indexOf("shadow: true\n")>0); Assert.assertTrue(ambassadorConfig.indexOf("prefix: /seldon/default/example/\n")>0); - Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v0")==4); + Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v1")==4); } @Test @@ -79,7 +79,7 @@ public void checkAmbassadorHeader() throws IOException, SeldonDeploymentExceptio System.out.println(ambassadorConfig); Assert.assertTrue(ambassadorConfig.indexOf(" location: london\n")>0); Assert.assertTrue(ambassadorConfig.indexOf("prefix: /seldon/default/example/\n")>0); - Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v0")==4); + Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v1")==4); } @Test @@ -97,7 +97,7 @@ public void checkAmbassadorCustomConfig() throws IOException, SeldonDeploymentEx System.out.println(ambassadorConfig); Assert.assertTrue(ambassadorConfig.indexOf("1234")==0); Assert.assertFalse(ambassadorConfig.indexOf("prefix: /seldon/default/example/\n")>0); - Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v0")==-1); + Assert.assertTrue(ambassadorConfig.indexOf("apiVersion: ambassador/v1")==-1); } } diff --git a/doc/source/graph/ambassador.md b/doc/source/graph/ambassador.md index 873f465a3e..915fa284c7 100644 --- a/doc/source/graph/ambassador.md +++ b/doc/source/graph/ambassador.md @@ -74,7 +74,7 @@ To understand more about the Ambassador configuration for this see [their docs o The above discussed configurations should cover most cases but there maybe a case where you want to have a very particular Ambassador configuration under your control. You can acheieve this by adding your confguration as an annotation to your Seldon Deployment resource. * `seldon.io/ambassador-config:` : The custom ambassador configuration - * Example: `"seldon.io/ambassador-config":"apiVersion: ambassador/v0\nkind: Mapping\nname: seldon_example_rest_mapping\nprefix: /mycompany/ml/\nservice: production-model-example.seldon:8000\ntimeout_ms: 3000"` + * Example: `"seldon.io/ambassador-config":"apiVersion: ambassador/v1\nkind: Mapping\nname: seldon_example_rest_mapping\nprefix: /mycompany/ml/\nservice: production-model-example.seldon:8000\ntimeout_ms: 3000"` A worked example for [custom Ambassador config](../examples/ambassador_custom.html) is provided. diff --git a/doc/source/workflow/troubleshooting.md b/doc/source/workflow/troubleshooting.md index e8cb2555ae..264b35aa8b 100644 --- a/doc/source/workflow/troubleshooting.md +++ b/doc/source/workflow/troubleshooting.md @@ -62,6 +62,8 @@ Check if the pods are running successfully. If your model is running and you are using Ambassador for ingress and are having problems check the diagnostics page of Ambassador. See [here](https://www.getambassador.io/reference/diagnostics/). You can then fnd out what path your model can be found under to ensure the URL you are using is correct. +If your ambassador isn't running at all then check the pod logs with `kubectl logs `. Note that if ambassador is installed with cluster-wide scope then its rbac should also not be namespaced, otherwise a there will be a permissions error. + ## I get 500s when calling my model over the API Check the logs of your running model pods. diff --git a/examples/ambassador/custom/model_custom_ambassador.json b/examples/ambassador/custom/model_custom_ambassador.json index 4ea6a1c9dc..c6cc201213 100644 --- a/examples/ambassador/custom/model_custom_ambassador.json +++ b/examples/ambassador/custom/model_custom_ambassador.json @@ -10,7 +10,7 @@ "spec": { "name": "production-model", "annotations": { - "seldon.io/ambassador-config":"apiVersion: ambassador/v0\nkind: Mapping\nname: seldon_example_rest_mapping\nprefix: /mycompany/ml/\nservice: production-model-example.seldon:8000\ntimeout_ms: 3000" + "seldon.io/ambassador-config":"apiVersion: ambassador/v1\nkind: Mapping\nname: seldon_example_rest_mapping\nprefix: /mycompany/ml/\nservice: production-model-example.seldon:8000\ntimeout_ms: 3000" }, "predictors": [ { From 561157c5d2e12aea482a945e7f94dba4217a6085 Mon Sep 17 00:00:00 2001 From: ryandawsonuk Date: Thu, 4 Apr 2019 18:27:40 +0100 Subject: [PATCH 2/4] get tests to wait for ambassador to handle grpc --- testing/scripts/seldon_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py index 4309fd5929..272d260150 100644 --- a/testing/scripts/seldon_utils.py +++ b/testing/scripts/seldon_utils.py @@ -101,7 +101,7 @@ def rest_request_ambassador_auth(deploymentName,namespace,username,password,endp auth=HTTPBasicAuth(username, password)) return response -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=100000, stop_max_attempt_number=9) def grpc_request_ambassador(deploymentName,namespace,endpoint="localhost:8004",data_size=5,rows=1,data=None): if data is None: shape, arr = create_random_data(data_size,rows) From 976c9e69aea9943112d7ad56f5256eaa1903a774 Mon Sep 17 00:00:00 2001 From: ryandawsonuk Date: Fri, 5 Apr 2019 17:08:21 +0100 Subject: [PATCH 3/4] run ambassador as root rather than downgrade --- helm-charts/seldon-core/values.yaml | 5 +++++ testing/scripts/k8s_utils.py | 4 ++-- testing/scripts/seldon_utils.py | 10 +++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/helm-charts/seldon-core/values.yaml b/helm-charts/seldon-core/values.yaml index 77509d5c22..22f9723976 100644 --- a/helm-charts/seldon-core/values.yaml +++ b/helm-charts/seldon-core/values.yaml @@ -1,5 +1,7 @@ ambassador: enabled: false + image: + tag: 0.52.1 replicaCount: 1 resources: limits: @@ -25,6 +27,9 @@ ambassador: rbac: create: true namespaced: true + securityContext: + runAsUser: 0 + runAsGroup: 0 # scope will be cluster wide unless below is set # env: # AMBASSADOR_SINGLE_NAMESPACE: "true" diff --git a/testing/scripts/k8s_utils.py b/testing/scripts/k8s_utils.py index fb1842d806..a15597dc42 100644 --- a/testing/scripts/k8s_utils.py +++ b/testing/scripts/k8s_utils.py @@ -94,7 +94,7 @@ def create_seldon_clusterwide_ksonnet(request): wait_seldon_ready() setup_finalizer_ksonnet(request) -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def port_forward(request): print("Setup: Port forward") p1 = Popen("kubectl port-forward $(kubectl get pods -n seldon -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].metadata.name}') -n seldon 8002:8080",stdout=subprocess.PIPE,shell=True, preexec_fn=os.setsid) @@ -120,7 +120,7 @@ def fin(): request.addfinalizer(fin) -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def port_forward_docker_repo(request): print("port-forward docker") p1 = Popen("POD_NAME=$(kubectl get pods -l app=docker-private-registry -n default |sed -e '1d'|awk '{print $1}') && kubectl port-forward ${POD_NAME} 5000:5000 -n default",stdout=subprocess.PIPE,shell=True, preexec_fn=os.setsid) diff --git a/testing/scripts/seldon_utils.py b/testing/scripts/seldon_utils.py index 272d260150..e79b81638e 100644 --- a/testing/scripts/seldon_utils.py +++ b/testing/scripts/seldon_utils.py @@ -26,7 +26,7 @@ def get_token(oauth_key,oauth_secret,namespace,endpoint): token = response.json()["access_token"] return token -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def rest_request_api_gateway(oauth_key,oauth_secret,namespace,endpoint="localhost:8002",data_size=5,rows=1,data=None): token = get_token(oauth_key,oauth_secret,namespace,endpoint) if data is None: @@ -42,7 +42,7 @@ def rest_request_api_gateway(oauth_key,oauth_secret,namespace,endpoint="localhos json=payload) return response -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint="localhost:8002",grpc_endpoint="localhost:8003",data_size=5,rows=1,data=None): token = get_token(oauth_key,oauth_secret,namespace,rest_endpoint) if data is None: @@ -63,7 +63,7 @@ def grpc_request_api_gateway(oauth_key,oauth_secret,namespace,rest_endpoint="loc response = stub.Predict(request=request,metadata=metadata) return response -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def rest_request_ambassador(deploymentName,namespace,endpoint="localhost:8003",data_size=5,rows=1,data=None): if data is None: shape, arr = create_random_data(data_size,rows) @@ -81,7 +81,7 @@ def rest_request_ambassador(deploymentName,namespace,endpoint="localhost:8003",d json=payload) return response -@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=7) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def rest_request_ambassador_auth(deploymentName,namespace,username,password,endpoint="localhost:8003",data_size=5,rows=1,data=None): if data is None: shape, arr = create_random_data(data_size,rows) @@ -101,7 +101,7 @@ def rest_request_ambassador_auth(deploymentName,namespace,username,password,endp auth=HTTPBasicAuth(username, password)) return response -@retry(wait_exponential_multiplier=1000, wait_exponential_max=100000, stop_max_attempt_number=9) +@retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=5) def grpc_request_ambassador(deploymentName,namespace,endpoint="localhost:8004",data_size=5,rows=1,data=None): if data is None: shape, arr = create_random_data(data_size,rows) From 956c835319071c8dd20225ebba184d843ebe391f Mon Sep 17 00:00:00 2001 From: ryandawsonuk Date: Mon, 8 Apr 2019 10:25:45 +0100 Subject: [PATCH 4/4] update to new ambassador --- helm-charts/seldon-core/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/helm-charts/seldon-core/values.yaml b/helm-charts/seldon-core/values.yaml index 22f9723976..baea43512e 100644 --- a/helm-charts/seldon-core/values.yaml +++ b/helm-charts/seldon-core/values.yaml @@ -1,7 +1,7 @@ ambassador: enabled: false image: - tag: 0.52.1 + tag: 0.53.1 replicaCount: 1 resources: limits: