From 31eb7de64d956836691469ae8e3bdd8d78b34409 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 9 Jun 2018 20:10:22 +0100 Subject: [PATCH 01/20] initial commit for distributed graphs --- .../clustermanager/k8s/KubeCRDHandler.java | 2 + .../k8s/KubeCRDHandlerImpl.java | 23 +- .../k8s/SeldonDeploymentControllerImpl.java | 100 ++++++--- .../k8s/SeldonDeploymentOperatorImpl.java | 212 ++++++++++++------ .../engine/predictors/EnginePredictor.java | 2 +- .../engine/predictors/PredictorBean.java | 7 +- .../predictors/RandomABTestUnitTest.java | 2 +- .../predictors/SimpleModelUnitTest.java | 6 +- proto/seldon_deployment.proto | 2 +- 9 files changed, 251 insertions(+), 105 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java index 16d36a693f..8105903d40 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandler.java @@ -16,6 +16,7 @@ package io.seldon.clustermanager.k8s; import io.kubernetes.client.models.ExtensionsV1beta1DeploymentList; +import io.kubernetes.client.models.V1ServiceList; import io.seldon.protos.DeploymentProtos.SeldonDeployment; public interface KubeCRDHandler { @@ -23,4 +24,5 @@ public interface KubeCRDHandler { public void updateSeldonDeployment(SeldonDeployment mlDep); public SeldonDeployment getSeldonDeployment(String name); public ExtensionsV1beta1DeploymentList getOwnedDeployments(String seldonDeploymentName); + public V1ServiceList getOwnedServices(String seldonDeploymentName); } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java index a56d9ead5e..ab53c2a5f9 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java @@ -29,9 +29,11 @@ import io.kubernetes.client.ApiClient; import io.kubernetes.client.ApiException; +import io.kubernetes.client.apis.CoreV1Api; import io.kubernetes.client.apis.CustomObjectsApi; import io.kubernetes.client.apis.ExtensionsV1beta1Api; import io.kubernetes.client.models.ExtensionsV1beta1DeploymentList; +import io.kubernetes.client.models.V1ServiceList; import io.kubernetes.client.proto.Meta.ObjectMeta; import io.kubernetes.client.util.Config; import io.seldon.clustermanager.ClusterManagerProperites; @@ -115,7 +117,7 @@ public ExtensionsV1beta1DeploymentList getOwnedDeployments(String seldonDeployme { ApiClient client = Config.defaultClient(); ExtensionsV1beta1Api api = new ExtensionsV1beta1Api(client); - ExtensionsV1beta1DeploymentList l = api.listNamespacedDeployment(namespace, null, null, null, false, Constants.LABEL_SELDON_ID+"="+seldonDeploymentName, 1, null, null, false); + ExtensionsV1beta1DeploymentList l = api.listNamespacedDeployment(namespace, null, null, null, false, Constants.LABEL_SELDON_ID+"="+seldonDeploymentName, null, null, null, false); return l; } catch (IOException e) { logger.error("Failed to get deployment list for "+seldonDeploymentName,e); @@ -125,6 +127,25 @@ public ExtensionsV1beta1DeploymentList getOwnedDeployments(String seldonDeployme return null; } } + + @Override + public V1ServiceList getOwnedServices(String seldonDeploymentName) { + try + { + ApiClient client = Config.defaultClient(); + io.kubernetes.client.apis.CoreV1Api api = new CoreV1Api(client); + V1ServiceList l = api.listNamespacedService(namespace, null, null, null, false, Constants.LABEL_SELDON_ID+"="+seldonDeploymentName, null, null, null, null); + return l; + } catch (IOException e) { + logger.error("Failed to get deployment list for "+seldonDeploymentName,e); + return null; + } catch (ApiException e) { + logger.error("Failed to get deployment list for "+seldonDeploymentName,e); + return null; + } + } + + } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java index aa38514546..638a4a81ae 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentControllerImpl.java @@ -31,6 +31,8 @@ import io.kubernetes.client.ProtoClient.ObjectOrStatus; import io.kubernetes.client.models.ExtensionsV1beta1Deployment; import io.kubernetes.client.models.ExtensionsV1beta1DeploymentList; +import io.kubernetes.client.models.V1Service; +import io.kubernetes.client.models.V1ServiceList; import io.kubernetes.client.proto.Meta.DeleteOptions; import io.kubernetes.client.proto.V1.Service; import io.kubernetes.client.proto.V1beta1Extensions.Deployment; @@ -107,7 +109,7 @@ private void createDeployments(ProtoClient client,String namespace,List getDeploymentNames(List deployments) { - Set names = new HashSet<>(); + Set names = new HashSet<>(); for(Deployment d : deployments) names.add(d.getMetadata().getName()); return names; @@ -137,37 +139,72 @@ private void removeDeployments(ProtoClient client,String namespace,SeldonDeploym } } - private void createService(ProtoClient client,String namespace,Service service) throws ApiException, IOException, SeldonDeploymentException + private void removeServices(ProtoClient client,String namespace,SeldonDeployment seldonDeployment,List services) throws ApiException, IOException, SeldonDeploymentException { - final String serviceApiPath = "/api/v1/namespaces/{namespace}/services/{name}" - .replaceAll("\\{" + "name" + "\\}", client.getApiClient().escapeString(service.getMetadata().getName())) - .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); - ObjectOrStatus os = client.list(Service.newBuilder(),serviceApiPath); - if (os.status != null) - { - if (os.status.getCode() == 404) - { - String serviceCreateApiPath = "/api/v1/namespaces/{namespace}/services" - .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); - os = client.create(service, serviceCreateApiPath, "v1", "Service"); - if (os.status != null) - { - logger.error("Error creating service "+ProtoBufUtils.toJson(os.status)); - throw new SeldonDeploymentException("Failed to create service "+service.getMetadata().getName()); + Set names = getServiceNames(services); + V1ServiceList svcList = crdHandler.getOwnedServices(seldonDeployment.getSpec().getName()); + for(V1Service s : svcList.getItems()) + { + if (!names.contains(s.getMetadata().getName())) + { + final String deleteApiPath = "/apis/v1/namespaces/{namespace}/services/{name}" + .replaceAll("\\{" + "name" + "\\}", client.getApiClient().escapeString(s.getMetadata().getName())) + .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); + DeleteOptions options = DeleteOptions.newBuilder().setPropagationPolicy("Foreground").build(); + ObjectOrStatus os = client.delete(Deployment.newBuilder(),deleteApiPath,options); + if (os.status != null) { + logger.error("Error deleting deployment:"+ProtoBufUtils.toJson(os.status)); + throw new SeldonDeploymentException("Failed to delete service "+s.getMetadata().getName()); } - else - { - logger.debug("Created service:"+ProtoBufUtils.toJson(os.object)); - } - } - else - { - logger.error("Error listing service:"+ProtoBufUtils.toJson(os.status)); - throw new SeldonDeploymentException("Failed to list service "+service.getMetadata().getName()); - } - } - else - logger.debug("No creating service as already exists "+service.getMetadata().getName()); + else { + logger.debug("Deleted deployment:"+ProtoBufUtils.toJson(os.object)); + } + } + } + } + + private Set getServiceNames(List services) + { + Set names = new HashSet<>(); + for(Service s : services) + names.add(s.getMetadata().getName()); + return names; + } + + private void createServices(ProtoClient client,String namespace,List services) throws ApiException, IOException, SeldonDeploymentException + { + for(Service service : services) + { + final String serviceApiPath = "/api/v1/namespaces/{namespace}/services/{name}" + .replaceAll("\\{" + "name" + "\\}", client.getApiClient().escapeString(service.getMetadata().getName())) + .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); + ObjectOrStatus os = client.list(Service.newBuilder(),serviceApiPath); + if (os.status != null) + { + if (os.status.getCode() == 404) + { + String serviceCreateApiPath = "/api/v1/namespaces/{namespace}/services" + .replaceAll("\\{" + "namespace" + "\\}", client.getApiClient().escapeString(namespace)); + os = client.create(service, serviceCreateApiPath, "v1", "Service"); + if (os.status != null) + { + logger.error("Error creating service "+ProtoBufUtils.toJson(os.status)); + throw new SeldonDeploymentException("Failed to create service "+service.getMetadata().getName()); + } + else + { + logger.debug("Created service:"+ProtoBufUtils.toJson(os.object)); + } + } + else + { + logger.error("Error listing service:"+ProtoBufUtils.toJson(os.status)); + throw new SeldonDeploymentException("Failed to list service "+service.getMetadata().getName()); + } + } + else + logger.debug("No creating service as already exists "+service.getMetadata().getName()); + } } private String getNamespace(SeldonDeployment d) @@ -207,7 +244,8 @@ public void createOrReplaceSeldonDeployment(SeldonDeployment mlDep) { String namespace = getNamespace(mlDep2); createDeployments(client, namespace, resources.deployments); removeDeployments(client, namespace, mlDep2, resources.deployments); - createService(client, namespace, resources.service); + createServices(client, namespace, resources.services); + removeServices(client,namespace, mlDep2, resources.services); if (!mlDep.getSpec().equals(mlDep2.getSpec())) { logger.debug("Pushing updated SeldonDeployment "+mlDep2.getMetadata().getName()+" back to kubectl"); diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index 64f8847c6c..c7be3e25fc 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -78,6 +78,11 @@ public SeldonDeploymentOperatorImpl(ClusterManagerProperites clusterManagerPrope this.clusterManagerProperites = clusterManagerProperites; } + //private String getContainerServiceName(SeldonDeployment dep,PredictorSpec p,String containerName) + //{ + // return dep.getSpec().getName() + "_" + p.getName() + "_" + containerName; + //} + private static String getEngineEnvVarJson(Message protoMessage) throws SeldonDeploymentException { String retVal; @@ -139,7 +144,6 @@ private V1.Container createEngineContainer(SeldonDeployment dep,PredictorSpec pr } - ; private Set getEnvNamesProto(List envs) { Set s = new HashSet<>(); @@ -285,7 +289,7 @@ private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int idx,St return c2Builder.build(); } - private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder,V1.Container container) + private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder,V1.Container container,String containerHostName) { if (puBuilder.getName().equals(container.getName())) { @@ -296,45 +300,65 @@ private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder, { b.setServicePort(p.getContainerPort()); b.setType(Endpoint.EndpointType.REST); - b.setServiceHost("0.0.0.0"); //assumes localhost at present + b.setServiceHost(containerHostName); return; } else if ("grpc".equals(p.getName())) { b.setServicePort(p.getContainerPort()); b.setType(Endpoint.EndpointType.GRPC); - b.setServiceHost("0.0.0.0"); //assumes localhost at present + b.setServiceHost(containerHostName); return; } } } else { for(int i=0;i 0 ? "0.0.0.0" : serviceName ; + updatePredictiveUnitBuilderByName(mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getGraphBuilder(),c2,containerHostName); + cIdx++; + } } - idx++; - } + } return mlBuilder.build(); } @@ -347,14 +371,15 @@ private void checkPredictiveUnitsMicroservices(PredictiveUnit pu,PredictorSpec p pu.getImplementation() == PredictiveUnitImplementation.UNKNOWN_IMPLEMENTATION) { boolean found = false; - for(V1.Container c : p.getComponentSpec().getSpec().getContainersList()) - { - if (c.getName().equals(pu.getName())) - { - found = true; - break; - } - } + for(V1.PodTemplateSpec spec : p.getComponentSpecsList()) + for(V1.Container c : spec.getSpec().getContainersList()) + { + if (c.getName().equals(pu.getName())) + { + found = true; + break; + } + } if (!found) { throw new SeldonDeploymentException("Can't find container for predictive unit with name "+pu.getName()); @@ -397,6 +422,8 @@ public String getKubernetesDeploymentName(String deploymentName,String predictor return deploymentName + "-" + predictorName; } + + private V1OwnerReference getOwnerReferenceOld(SeldonDeployment mlDep) { return new V1OwnerReference() @@ -437,53 +464,106 @@ private String getAmbassadorAnnotation(SeldonDeployment mlDep,String serviceName return restMapping + grpcMapping; } + + private void addServicePorts(PredictiveUnit pu,String serviceName,ServiceSpec.Builder svcSpecBuilder) + { + if (pu.hasEndpoint()) + { + Endpoint e = pu.getEndpoint(); + if (e.getServiceHost().equals(serviceName)) + { + svcSpecBuilder.addPorts(ServicePort.newBuilder() + .setProtocol("TCP") + .setPort(e.getServicePort()) + .setTargetPort(IntOrString.newBuilder().setIntVal(e.getServicePort())) + .setName(e.getType().toString().toLowerCase()) + ); + } + } + for(int i=0;i deployments = new ArrayList<>(); + List services = new ArrayList<>(); // for each predictor Create/replace deployment String serviceLabel = mlDep.getSpec().getName(); - for(PredictorSpec p : mlDep.getSpec().getPredictorsList()) + for(int pbIdx=0;pbIdx deployments; - Service service; + List services; - public DeploymentResources(List deployments, Service service) { + public DeploymentResources(List deployments, List services) { super(); this.deployments = deployments; - this.service = service; + this.services = services; } diff --git a/engine/src/main/java/io/seldon/engine/predictors/EnginePredictor.java b/engine/src/main/java/io/seldon/engine/predictors/EnginePredictor.java index ab3fdd92a6..4496f3ec47 100644 --- a/engine/src/main/java/io/seldon/engine/predictors/EnginePredictor.java +++ b/engine/src/main/java/io/seldon/engine/predictors/EnginePredictor.java @@ -133,7 +133,7 @@ private static PredictorSpec buildDefaultPredictorSpec() { //@formatter:off PredictorSpec.Builder predictorSpecBuilder = PredictorSpec.newBuilder() .setName("basic-predictor") - .setComponentSpec(PodTemplateSpec.newBuilder()); + .addComponentSpecs(PodTemplateSpec.newBuilder()); //@formatter:on { // Add predictorGraph diff --git a/engine/src/main/java/io/seldon/engine/predictors/PredictorBean.java b/engine/src/main/java/io/seldon/engine/predictors/PredictorBean.java index e9b136fe4b..e5b60f0ef5 100644 --- a/engine/src/main/java/io/seldon/engine/predictors/PredictorBean.java +++ b/engine/src/main/java/io/seldon/engine/predictors/PredictorBean.java @@ -33,7 +33,7 @@ import io.seldon.protos.DeploymentProtos.PredictorSpec; import io.seldon.protos.PredictionProtos.Feedback; import io.seldon.protos.PredictionProtos.SeldonMessage; - +import io.kubernetes.client.proto.V1; import io.kubernetes.client.proto.V1.Container; @@ -70,8 +70,9 @@ public PredictorState predictorStateFromPredictorSpec(PredictorSpec predictorSpe PredictiveUnit rootUnit = predictorSpec.getGraph(); Map containersMap = new HashMap(); - for (Container container : predictorSpec.getComponentSpec().getSpec().getContainersList()){ - containersMap.put(container.getName(), container); + for(V1.PodTemplateSpec spec : predictorSpec.getComponentSpecsList()) + for (Container container : spec.getSpec().getContainersList()){ + containersMap.put(container.getName(), container); } PredictiveUnitState rootState = new PredictiveUnitState(rootUnit,containersMap); diff --git a/engine/src/test/java/io/seldon/engine/predictors/RandomABTestUnitTest.java b/engine/src/test/java/io/seldon/engine/predictors/RandomABTestUnitTest.java index 6dfcb2f2e0..64bc3c48ab 100644 --- a/engine/src/test/java/io/seldon/engine/predictors/RandomABTestUnitTest.java +++ b/engine/src/test/java/io/seldon/engine/predictors/RandomABTestUnitTest.java @@ -46,7 +46,7 @@ public void simpleTest() throws InterruptedException, ExecutionException, Invali predictorSpecBuilder.setName("p1"); predictorSpecBuilder.setReplicas(1); - predictorSpecBuilder.setComponentSpec(PodTemplateSpec.newBuilder()); + predictorSpecBuilder.addComponentSpecs(PodTemplateSpec.newBuilder()); PredictiveUnit.Builder predictiveUnitBuilder = PredictiveUnit.newBuilder(); predictiveUnitBuilder.setName("1"); diff --git a/engine/src/test/java/io/seldon/engine/predictors/SimpleModelUnitTest.java b/engine/src/test/java/io/seldon/engine/predictors/SimpleModelUnitTest.java index d628392012..14d21b0186 100644 --- a/engine/src/test/java/io/seldon/engine/predictors/SimpleModelUnitTest.java +++ b/engine/src/test/java/io/seldon/engine/predictors/SimpleModelUnitTest.java @@ -49,7 +49,7 @@ public void simpleTest() throws InterruptedException, ExecutionException, Invali PredictorSpecBuilder.setName("p1"); // PredictorSpecBuilder.setRoot("1"); PredictorSpecBuilder.setReplicas(1); - PredictorSpecBuilder.setComponentSpec(PodTemplateSpec.newBuilder()); + PredictorSpecBuilder.addComponentSpecs(PodTemplateSpec.newBuilder()); PredictiveUnit.Builder PredictiveUnitBuilder = PredictiveUnit.newBuilder(); PredictiveUnitBuilder.setName("1"); @@ -88,7 +88,7 @@ public void simpleTestWithImageNoVersion() throws InterruptedException, Executio final String imageName = "myimage"; PodTemplateSpec.Builder ptsBuilder = PodTemplateSpec.newBuilder().setSpec(PodSpec.newBuilder().addContainers(Container.newBuilder().setImage(imageName).setName("1"))); - PredictorSpecBuilder.setComponentSpec(ptsBuilder); + PredictorSpecBuilder.addComponentSpecs(ptsBuilder); PredictiveUnit.Builder PredictiveUnitBuilder = PredictiveUnit.newBuilder(); PredictiveUnitBuilder.setName("1"); @@ -129,7 +129,7 @@ public void simpleTestWithImageVersion() throws InterruptedException, ExecutionE final String imageVersion = "0.1"; PodTemplateSpec.Builder ptsBuilder = PodTemplateSpec.newBuilder().setSpec(PodSpec.newBuilder().addContainers(Container.newBuilder().setImage(imageName+":"+imageVersion).setName("1"))); - PredictorSpecBuilder.setComponentSpec(ptsBuilder); + PredictorSpecBuilder.addComponentSpecs(ptsBuilder); PredictiveUnit.Builder PredictiveUnitBuilder = PredictiveUnit.newBuilder(); PredictiveUnitBuilder.setName("1"); diff --git a/proto/seldon_deployment.proto b/proto/seldon_deployment.proto index c1ae6e1f19..a8c84894fb 100644 --- a/proto/seldon_deployment.proto +++ b/proto/seldon_deployment.proto @@ -44,7 +44,7 @@ message DeploymentSpec { message PredictorSpec { required string name = 1; // A unique name not used by any other predictor in the deployment. required PredictiveUnit graph = 2; // A graph describing how the predictive units are connected together. - required k8s.io.api.core.v1.PodTemplateSpec componentSpec = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. + repeated k8s.io.api.core.v1.PodTemplateSpec componentSpecs = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. Can be split over 1 or more PodTemplateSpecs. optional int32 replicas = 4; // The number of replicas of the predictor to create. map annotations = 5; // Arbitrary annotations. optional k8s.io.api.core.v1.ResourceRequirements engineResources = 6; // Optional set of resources for the Seldon engine which is added to each Predictor graph to manage the request/response flow From 5095da7c146f55dcb2283cac8076754dd92c643b Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 10 Jun 2018 14:52:26 +0100 Subject: [PATCH 02/20] initial updates from testing. Changes to move to v1alpha2 api version --- .../AuthorizationServerConfiguration.java | 3 ++- .../seldon/apife/grpc/SeldonGrpcServer.java | 3 ++- .../seldon/apife/k8s/DeploymentWatcher.java | 7 +++++- .../k8s/KubeCRDHandlerImpl.java | 2 +- .../k8s/SeldonDeploymentOperator.java | 2 +- .../k8s/SeldonDeploymentOperatorImpl.java | 25 ++++++++----------- .../k8s/SeldonDeploymentWatcher.java | 2 +- .../k8s/SeldonDeploymentDefaultingTest.java | 24 +++++++++--------- .../test/resources/model_invalid_graph.json | 4 +-- .../resources/model_invalid_no_method.json | 4 +-- .../src/test/resources/model_simple.json | 4 +-- .../src/test/resources/model_simple_grpc.json | 4 +-- .../templates/seldon-deployment-crd.json | 2 +- notebooks/resources/model.json | 6 ++--- notebooks/resources/random_ab_test.json | 15 +++++++---- 15 files changed, 58 insertions(+), 49 deletions(-) diff --git a/api-frontend/src/main/java/io/seldon/apife/config/AuthorizationServerConfiguration.java b/api-frontend/src/main/java/io/seldon/apife/config/AuthorizationServerConfiguration.java index b5ebb2544c..788a22f0fd 100644 --- a/api-frontend/src/main/java/io/seldon/apife/config/AuthorizationServerConfiguration.java +++ b/api-frontend/src/main/java/io/seldon/apife/config/AuthorizationServerConfiguration.java @@ -31,6 +31,7 @@ import io.seldon.apife.api.oauth.InMemoryClientDetailsService; import io.seldon.apife.deployments.DeploymentStore; +import io.seldon.apife.k8s.DeploymentWatcher; import io.seldon.protos.DeploymentProtos.DeploymentSpec; import io.seldon.protos.DeploymentProtos.SeldonDeployment; @@ -83,7 +84,7 @@ public void configure(ClientDetailsServiceConfigurer clients) throws Exception { String client_secret = System.getenv().get(TEST_CLIENT_SECRET); clientDetailsService.addClient(client_key,client_secret); SeldonDeployment dep = SeldonDeployment.newBuilder() - .setApiVersion("v1alpha1") + .setApiVersion(DeploymentWatcher.VERSION) .setKind("SeldonDeplyment") .setSpec(DeploymentSpec.newBuilder() .setName("localhost") diff --git a/api-frontend/src/main/java/io/seldon/apife/grpc/SeldonGrpcServer.java b/api-frontend/src/main/java/io/seldon/apife/grpc/SeldonGrpcServer.java index fc441ec499..4e4aa2a449 100644 --- a/api-frontend/src/main/java/io/seldon/apife/grpc/SeldonGrpcServer.java +++ b/api-frontend/src/main/java/io/seldon/apife/grpc/SeldonGrpcServer.java @@ -38,6 +38,7 @@ import io.seldon.apife.deployments.DeploymentsHandler; import io.seldon.apife.deployments.DeploymentsListener; import io.seldon.apife.exception.SeldonAPIException; +import io.seldon.apife.k8s.DeploymentWatcher; import io.seldon.protos.DeploymentProtos.DeploymentSpec; import io.seldon.protos.DeploymentProtos.Endpoint; import io.seldon.protos.DeploymentProtos.SeldonDeployment; @@ -179,7 +180,7 @@ private void blockUntilShutdown() throws InterruptedException { public static void main(String[] args) throws Exception { DeploymentStore store = new DeploymentStore(null,new InMemoryClientDetailsService()); SeldonDeployment dep = SeldonDeployment.newBuilder() - .setApiVersion("v1alpha1") + .setApiVersion(DeploymentWatcher.VERSION) .setKind("SeldonDeplyment") .setSpec(DeploymentSpec.newBuilder() .setName("0.0.0.0") diff --git a/api-frontend/src/main/java/io/seldon/apife/k8s/DeploymentWatcher.java b/api-frontend/src/main/java/io/seldon/apife/k8s/DeploymentWatcher.java index 14f4b62766..a2f8f56353 100644 --- a/api-frontend/src/main/java/io/seldon/apife/k8s/DeploymentWatcher.java +++ b/api-frontend/src/main/java/io/seldon/apife/k8s/DeploymentWatcher.java @@ -57,6 +57,11 @@ public class DeploymentWatcher implements DeploymentsHandler{ protected static Logger logger = LoggerFactory.getLogger(DeploymentWatcher.class.getName()); + public static final String GROUP = "machinelearning.seldon.io"; + public static final String VERSION = "v1alpha2"; + public static final String KIND_PLURAL = "seldondeployments"; + public static final String KIND = "SeldonDeployment"; + private ApiClient client; private int resourceVersion = 0; private int resourceVersionProcessed = 0; @@ -124,7 +129,7 @@ public int watchSeldonSeldonDeployments(int resourceVersion,int resourceVersionP CustomObjectsApi api = new CustomObjectsApi(); Watch watch = Watch.createWatch( client, - api.listNamespacedCustomObjectCall("machinelearning.seldon.io", "v1alpha1", namespace, "seldondeployments", null, null, rs, true, null, null), + api.listNamespacedCustomObjectCall(GROUP, VERSION, namespace, KIND_PLURAL, null, null, rs, true, null, null), new TypeToken>(){}.getType()); int maxResourceVersion = resourceVersion; diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java index ab53c2a5f9..146df3fd28 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/KubeCRDHandlerImpl.java @@ -45,7 +45,7 @@ public class KubeCRDHandlerImpl implements KubeCRDHandler { private final static Logger logger = LoggerFactory.getLogger(KubeCRDHandlerImpl.class); public static final String GROUP = "machinelearning.seldon.io"; - public static final String VERSION = "v1alpha1"; + public static final String VERSION = "v1alpha2"; public static final String KIND_PLURAL = "seldondeployments"; public static final String KIND = "SeldonDeployment"; diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java index b12b34c418..4fdc8a5f44 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java @@ -23,6 +23,6 @@ public interface SeldonDeploymentOperator { public SeldonDeployment defaulting(SeldonDeployment mlDep); public void validate(SeldonDeployment mlDep) throws SeldonDeploymentException; public DeploymentResources createResources(SeldonDeployment mlDep) throws SeldonDeploymentException; - public String getKubernetesDeploymentName(String depName,String predictorName); + public String getKubernetesDeploymentName(String depName,String predictorName,int podTemplateIdx); } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index c7be3e25fc..c5eeeed5ab 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -78,12 +78,7 @@ public SeldonDeploymentOperatorImpl(ClusterManagerProperites clusterManagerPrope this.clusterManagerProperites = clusterManagerProperites; } - //private String getContainerServiceName(SeldonDeployment dep,PredictorSpec p,String containerName) - //{ - // return dep.getSpec().getName() + "_" + p.getName() + "_" + containerName; - //} - - + private static String getEngineEnvVarJson(Message protoMessage) throws SeldonDeploymentException { String retVal; try { @@ -317,9 +312,15 @@ private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder, private String getPredictorServiceName(SeldonDeployment mlDep,int predictorIdx,int podTemplateIdx) { - return mlDep.getSpec().getName()+"_"+predictorIdx+"_"+podTemplateIdx; + return mlDep.getSpec().getName()+"-"+predictorIdx+"-"+podTemplateIdx; } + @Override + public String getKubernetesDeploymentName(String deploymentName,String predictorName,int podTemplateIdx) { + return deploymentName + "-" + predictorName+"-"+podTemplateIdx; + } + + @Override public SeldonDeployment defaulting(SeldonDeployment mlDep) { SeldonDeployment.Builder mlBuilder = SeldonDeployment.newBuilder(mlDep); @@ -353,7 +354,7 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) { { V1.Container c2 = this.updateContainer(c, findPredictiveUnitForContainer(mlDep.getSpec().getPredictors(pbIdx).getGraph(),c.getName()),cIdx,deploymentName,predictorName); mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getComponentSpecsBuilder(ptsIdx).getSpecBuilder().addContainers(cIdx, c2); - String containerHostName = ptsIdx > 0 ? "0.0.0.0" : serviceName ; + String containerHostName = ptsIdx == 0 ? "0.0.0.0" : serviceName ; updatePredictiveUnitBuilderByName(mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getGraphBuilder(),c2,containerHostName); cIdx++; } @@ -417,10 +418,6 @@ public void validate(SeldonDeployment mlDep) throws SeldonDeploymentException { } - @Override - public String getKubernetesDeploymentName(String deploymentName,String predictorName) { - return deploymentName + "-" + predictorName; - } @@ -481,7 +478,7 @@ private void addServicePorts(PredictiveUnit pu,String serviceName,ServiceSpec.Bu } } for(int i=0;i watch = Watch.createWatch( client, - api.listNamespacedCustomObjectCall("machinelearning.seldon.io", "v1alpha1", namespace, "seldondeployments", null, null, rs, true, null, null), + api.listNamespacedCustomObjectCall(KubeCRDHandlerImpl.GROUP, KubeCRDHandlerImpl.VERSION, namespace, KubeCRDHandlerImpl.KIND_PLURAL, null, null, rs, true, null, null), new TypeToken>(){}.getType()); int maxResourceVersion = resourceVersion; diff --git a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java index d716c0c7ca..03aa98def6 100644 --- a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java +++ b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java @@ -35,12 +35,12 @@ public void testDefaulting() throws IOException SeldonDeployment mlDep = SeldonDeploymentUtils.jsonToSeldonDeployment(jsonStr); SeldonDeployment mlDep2 = op.defaulting(mlDep); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasLivenessProbe()); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasReadinessProbe()); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasLifecycle()); - Assert.assertEquals("Incorrect number of environment variables in container",5,mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getEnvCount()); - Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getPortsCount()); - Assert.assertEquals("http",mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getPorts(0).getName()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasLivenessProbe()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasReadinessProbe()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasLifecycle()); + Assert.assertEquals("Incorrect number of environment variables in container",5,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getEnvCount()); + Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPortsCount()); + Assert.assertEquals("http",mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPorts(0).getName()); Assert.assertEquals(Endpoint.EndpointType.REST_VALUE,mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getType().getNumber()); Assert.assertEquals("0.0.0.0",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); } @@ -53,12 +53,12 @@ public void testDefaultingGrpc() throws IOException SeldonDeployment mlDep = SeldonDeploymentUtils.jsonToSeldonDeployment(jsonStr); SeldonDeployment mlDep2 = op.defaulting(mlDep); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasLivenessProbe()); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasReadinessProbe()); - Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).hasLifecycle()); - Assert.assertEquals("Incorrect number of environment variables in container",5,mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getEnvCount()); - Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getPortsCount()); - Assert.assertEquals("grpc",mlDep2.getSpec().getPredictors(0).getComponentSpec().getSpec().getContainers(0).getPorts(0).getName()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasLivenessProbe()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasReadinessProbe()); + Assert.assertTrue(mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).hasLifecycle()); + Assert.assertEquals("Incorrect number of environment variables in container",5,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getEnvCount()); + Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPortsCount()); + Assert.assertEquals("grpc",mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPorts(0).getName()); Assert.assertEquals(Endpoint.EndpointType.GRPC_VALUE,mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getType().getNumber()); Assert.assertEquals("0.0.0.0",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); } diff --git a/cluster-manager/src/test/resources/model_invalid_graph.json b/cluster-manager/src/test/resources/model_invalid_graph.json index dc64bcb1a8..c263e39ed0 100644 --- a/cluster-manager/src/test/resources/model_invalid_graph.json +++ b/cluster-manager/src/test/resources/model_invalid_graph.json @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier_", diff --git a/cluster-manager/src/test/resources/model_invalid_no_method.json b/cluster-manager/src/test/resources/model_invalid_no_method.json index 26d51dce76..4755ca8b93 100644 --- a/cluster-manager/src/test/resources/model_invalid_no_method.json +++ b/cluster-manager/src/test/resources/model_invalid_no_method.json @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpec": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/cluster-manager/src/test/resources/model_simple.json b/cluster-manager/src/test/resources/model_simple.json index 9987d30f76..e6b4979943 100644 --- a/cluster-manager/src/test/resources/model_simple.json +++ b/cluster-manager/src/test/resources/model_simple.json @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/cluster-manager/src/test/resources/model_simple_grpc.json b/cluster-manager/src/test/resources/model_simple_grpc.json index c0ed4c82fe..27d33e772b 100644 --- a/cluster-manager/src/test/resources/model_simple_grpc.json +++ b/cluster-manager/src/test/resources/model_simple_grpc.json @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs":[{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json b/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json index 182436618c..7f285d4e04 100644 --- a/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json +++ b/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json @@ -239,6 +239,6 @@ } } }, - "version": "v1alpha1" + "version": "v1alpha2" } } diff --git a/notebooks/resources/model.json b/notebooks/resources/model.json index 641ccd5199..b3145f5a90 100644 --- a/notebooks/resources/model.json +++ b/notebooks/resources/model.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "classifier", diff --git a/notebooks/resources/random_ab_test.json b/notebooks/resources/random_ab_test.json index 8f3c675453..0a12a9dd3b 100644 --- a/notebooks/resources/random_ab_test.json +++ b/notebooks/resources/random_ab_test.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -29,7 +29,12 @@ "memory": "1Mi" } } - }, + }], + "terminationGracePeriodSeconds": 20 + }}, + { + "spec":{ + "containers":[ { "image": "seldonio/mock_classifier:1.0", "imagePullPolicy": "IfNotPresent", @@ -42,8 +47,8 @@ } ], "terminationGracePeriodSeconds": 20 - } - }, + } + }], "name": "fx-market-predictor", "replicas": 1, "annotations": { From dd266c0ade661cfdf6a3f6ce2eebcb9ba81e214c Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 10 Jun 2018 18:34:05 +0100 Subject: [PATCH 03/20] update notebook k8s resources to v1alpha2 --- notebooks/resources/complex_graph.json | 26 ++++++++++++++----- .../resources/complex_graph_with_canary.json | 10 +++---- notebooks/resources/ensemble.json | 6 ++--- notebooks/resources/epsilon_greedy.json | 6 ++--- notebooks/resources/feature_transform.json | 6 ++--- .../resources/loadtest_simple_model.json | 6 ++--- notebooks/resources/model_grpc.json | 6 ++--- notebooks/resources/model_invalid1.json | 6 ++--- notebooks/resources/model_invalid2.json | 6 ++--- notebooks/resources/model_with_canary.json | 10 +++---- .../model_with_engine_resources.json | 6 ++--- notebooks/resources/outlier_detector.json | 13 +++++++--- 12 files changed, 62 insertions(+), 45 deletions(-) diff --git a/notebooks/resources/complex_graph.json b/notebooks/resources/complex_graph.json index 479454be2a..699f1222b7 100644 --- a/notebooks/resources/complex_graph.json +++ b/notebooks/resources/complex_graph.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -30,7 +30,10 @@ "cpu": "0.1" } } - }, + }]}}, + { + "spec": { + "containers": [ { "image": "seldonio/mock_classifier:1.0", "imagePullPolicy": "IfNotPresent", @@ -41,7 +44,10 @@ "cpu": "0.1" } } - }, + }]}}, + { + "spec": { + "containers": [ { "image": "seldonio/mock_classifier:1.0", "imagePullPolicy": "IfNotPresent", @@ -52,7 +58,10 @@ "cpu": "0.1" } } - }, + }]}}, + { + "spec": { + "containers": [ { "image": "seldonio/mock_outlier_detector:1.0", "imagePullPolicy": "IfNotPresent", @@ -63,7 +72,10 @@ "cpu": "0.1" } } - }, + }]}}, + { + "spec": { + "containers": [ { "image": "seldonio/mock_transformer:1.0", "imagePullPolicy": "IfNotPresent", @@ -78,7 +90,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "fx-market-predictor", "replicas": 1, "annotations": { diff --git a/notebooks/resources/complex_graph_with_canary.json b/notebooks/resources/complex_graph_with_canary.json index 060b4dfa69..88d4b3780e 100644 --- a/notebooks/resources/complex_graph_with_canary.json +++ b/notebooks/resources/complex_graph_with_canary.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -78,7 +78,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "fx-market-predictor", "replicas": 1, "annotations": { @@ -144,7 +144,7 @@ } }, { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -172,7 +172,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "fx-market-predictor-canary", "replicas": 1, "annotations": { diff --git a/notebooks/resources/ensemble.json b/notebooks/resources/ensemble.json index 9b6c697564..c90deb7e08 100644 --- a/notebooks/resources/ensemble.json +++ b/notebooks/resources/ensemble.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -53,7 +53,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "fx-market-predictor", "replicas": 1, "annotations": { diff --git a/notebooks/resources/epsilon_greedy.json b/notebooks/resources/epsilon_greedy.json index 7bb0ae7da3..4a9a6b74c9 100644 --- a/notebooks/resources/epsilon_greedy.json +++ b/notebooks/resources/epsilon_greedy.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -39,7 +39,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "multi-models-predictor", "replicas": 1, "annotations": { diff --git a/notebooks/resources/feature_transform.json b/notebooks/resources/feature_transform.json index 5a0a9e409f..ea25dd3b32 100644 --- a/notebooks/resources/feature_transform.json +++ b/notebooks/resources/feature_transform.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -43,7 +43,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "name": "transformer", "endpoint": { diff --git a/notebooks/resources/loadtest_simple_model.json b/notebooks/resources/loadtest_simple_model.json index 70883b5611..2a3e1d876a 100644 --- a/notebooks/resources/loadtest_simple_model.json +++ b/notebooks/resources/loadtest_simple_model.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,13 +17,13 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "stub", diff --git a/notebooks/resources/model_grpc.json b/notebooks/resources/model_grpc.json index f088751744..9fa5f8eec2 100644 --- a/notebooks/resources/model_grpc.json +++ b/notebooks/resources/model_grpc.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/notebooks/resources/model_invalid1.json b/notebooks/resources/model_invalid1.json index 5d75795a71..4d6ef9a749 100644 --- a/notebooks/resources/model_invalid1.json +++ b/notebooks/resources/model_invalid1.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/notebooks/resources/model_invalid2.json b/notebooks/resources/model_invalid2.json index dc64bcb1a8..dd3b85acee 100644 --- a/notebooks/resources/model_invalid2.json +++ b/notebooks/resources/model_invalid2.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mean-classifier_", diff --git a/notebooks/resources/model_with_canary.json b/notebooks/resources/model_with_canary.json index d63938d2db..0faef4801b 100644 --- a/notebooks/resources/model_with_canary.json +++ b/notebooks/resources/model_with_canary.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -16,7 +16,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -31,7 +31,7 @@ } ] } - }, + }], "graph": { "children": [], "name": "mean-classifier", @@ -44,7 +44,7 @@ "replicas": 1 }, { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -59,7 +59,7 @@ } ] } - }, + }], "graph": { "children": [], "name": "mean-classifier", diff --git a/notebooks/resources/model_with_engine_resources.json b/notebooks/resources/model_with_engine_resources.json index 100b1cee4b..a3a53fc646 100644 --- a/notebooks/resources/model_with_engine_resources.json +++ b/notebooks/resources/model_with_engine_resources.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "classifier", diff --git a/notebooks/resources/outlier_detector.json b/notebooks/resources/outlier_detector.json index a94ca7e374..8ad36af795 100644 --- a/notebooks/resources/outlier_detector.json +++ b/notebooks/resources/outlier_detector.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -41,7 +41,12 @@ "cpu": "0.1" } } - }, + }], + "terminationGracePeriodSeconds": 20 + }}, + { + "spec": { + "containers": [ { "image": "seldonio/outlier_mahalanobis:0.2", "imagePullPolicy": "IfNotPresent", @@ -56,7 +61,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "name": "fx-market-predictor", "replicas": 1, "annotations": { From 553ea88b24e390df5ede1325dda45c7cd91526e6 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Wed, 13 Jun 2018 08:47:48 +0100 Subject: [PATCH 04/20] updates to allow engine to use svc names for all calls --- cluster-manager/README.txt | 4 + .../k8s/SeldonDeploymentOperatorImpl.java | 198 ++++++++++------ .../k8s/SeldonDeploymentDefaultingTest.java | 4 +- notebooks/kubectl_demo_minikube.ipynb | 224 +++++++++++++++--- notebooks/resources/random_ab_test.json | 11 +- 5 files changed, 332 insertions(+), 109 deletions(-) create mode 100644 cluster-manager/README.txt diff --git a/cluster-manager/README.txt b/cluster-manager/README.txt new file mode 100644 index 0000000000..bea5f9a39e --- /dev/null +++ b/cluster-manager/README.txt @@ -0,0 +1,4 @@ +Local testing: + +export SELDON_CLUSTER_MANAGER_POD_NAMESPACE=seldon +export ENGINE_CONTAINER_IMAGE_AND_VERSION=seldonio/engine:0.1.8-SNAPSHOT diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index c5eeeed5ab..dfd5c51d47 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -17,8 +17,11 @@ import java.util.ArrayList; import java.util.Base64; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.StringJoiner; @@ -187,7 +190,7 @@ private PredictiveUnit findPredictiveUnitForContainer(PredictiveUnit unit,String } } - private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int idx,String deploymentName,String predictorName) + private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int portNum,String deploymentName,String predictorName) { V1.Container.Builder c2Builder = V1.Container.newBuilder(c); @@ -199,8 +202,8 @@ private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int idx,St { if (pu.getEndpoint().getType() == Endpoint.EndpointType.REST) { - c2Builder.addPorts(ContainerPort.newBuilder().setName("http").setContainerPort(clusterManagerProperites.getPuContainerPortBase() + idx)); - containerPort = clusterManagerProperites.getPuContainerPortBase() + idx; + c2Builder.addPorts(ContainerPort.newBuilder().setName("http").setContainerPort(portNum)); + containerPort = portNum; if (!c.hasLivenessProbe()) { @@ -222,8 +225,8 @@ private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int idx,St } else { - c2Builder.addPorts(ContainerPort.newBuilder().setName("grpc").setContainerPort(clusterManagerProperites.getPuContainerPortBase() + idx)); - containerPort = clusterManagerProperites.getPuContainerPortBase() + idx; + c2Builder.addPorts(ContainerPort.newBuilder().setName("grpc").setContainerPort(portNum)); + containerPort = portNum; if (!c.hasLivenessProbe()) { @@ -247,7 +250,7 @@ private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int idx,St } } else - containerPort = c.getPorts(0).getContainerPort(); + throw new UnsupportedOperationException(String.format("Found container port already set with http or grpc label. This is not presently allowed. Found port {}",containerPort)); // Add environment variable for the port used in case the model needs to access it final String ENV_PREDICTIVE_UNIT_SERVICE_PORT ="PREDICTIVE_UNIT_SERVICE_PORT"; @@ -310,9 +313,14 @@ private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder, } } - private String getPredictorServiceName(SeldonDeployment mlDep,int predictorIdx,int podTemplateIdx) + private String getPredictorServiceNameValue(SeldonDeployment mlDep,String predictorName,String containerName) { - return mlDep.getSpec().getName()+"-"+predictorIdx+"-"+podTemplateIdx; + return mlDep.getSpec().getName()+"-"+predictorName+"-"+containerName; + } + + private String getPredictorServiceNameKey(String containerName) + { + return LABEL_SELDON_APP+"-"+containerName; } @Override @@ -329,35 +337,37 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) { for(int pbIdx=0;pbIdx servicePortMap = new HashMap<>(); + int currentServicePortNum = clusterManagerProperites.getPuContainerPortBase(); for(int ptsIdx=0;ptsIdx services = new ArrayList<>(); // for each predictor Create/replace deployment String serviceLabel = mlDep.getSpec().getName(); + Set createdServices = new HashSet<>(); for(int pbIdx=0;pbIdx podLabel : spec.getMetadata().getLabelsMap().entrySet()) + //{ + // depMetaBuilder.put + //} + + for(V1.Container c : spec.getSpec().getContainersList()) { - depServiceLabelValue = getPredictorServiceName(mlDep, pbIdx, ptsIdx); + String containerServiceKey = getPredictorServiceNameKey(c.getName()); + String containerServiceValue = getPredictorServiceNameValue(mlDep, p.getName(), c.getName()); + podSpecBuilder.getSpecBuilder() .setTerminationGracePeriodSeconds(20); - //Add service - Service.Builder s = Service.newBuilder() - .setMetadata(ObjectMeta.newBuilder() - .setName(depServiceLabelValue) - .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP, depServiceLabelValue) - .putLabels("seldon-deployment-id", mlDep.getSpec().getName()) - .addOwnerReferences(ownerRef) - .putAnnotations("getambassador.io/config",getAmbassadorAnnotation(mlDep,serviceLabel)) - ); - ServiceSpec.Builder svcSpecBuilder = ServiceSpec.newBuilder(); - addServicePorts(p.getGraph(), depServiceLabelValue, svcSpecBuilder); - svcSpecBuilder.setType("ClusterIP") - .putSelector(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP,depServiceLabelValue); - - s.setSpec(svcSpecBuilder); - services.add(s.build()); + if (!createdServices.contains(containerServiceValue)) + { + //Add service + Service.Builder s = Service.newBuilder() + .setMetadata(ObjectMeta.newBuilder() + .setName(containerServiceValue) + .putLabels(containerServiceKey, containerServiceValue) + .putLabels("seldon-deployment-id", mlDep.getSpec().getName()) + .addOwnerReferences(ownerRef) + ); + ServiceSpec.Builder svcSpecBuilder = ServiceSpec.newBuilder(); + addServicePort(p.getGraph(), containerServiceValue, svcSpecBuilder); + svcSpecBuilder.setType("ClusterIP") + .putSelector(containerServiceKey,containerServiceValue); + + depMetaBuilder.putLabels(containerServiceKey, containerServiceValue); + s.setSpec(svcSpecBuilder); + services.add(s.build()); + } } - + + + + Deployment deployment = V1beta1Extensions.Deployment.newBuilder() - .setMetadata(ObjectMeta.newBuilder() - .setName(depName) - .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP, depServiceLabelValue) - .putLabels(Constants.LABEL_SELDON_ID, mlDep.getSpec().getName()) - .putLabels("app", depName) - .putLabels("version", "v1") //FIXME - .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_KEY, SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_VAL) - .addOwnerReferences(ownerRef) - ) + .setMetadata(depMetaBuilder) .setSpec(DeploymentSpec.newBuilder() .setTemplate(podSpecBuilder.build()) .setStrategy(DeploymentStrategy.newBuilder().setRollingUpdate(RollingUpdateDeployment.newBuilder().setMaxUnavailable(IntOrString.newBuilder().setType(1).setStrVal("10%")))) diff --git a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java index 03aa98def6..a6e402de68 100644 --- a/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java +++ b/cluster-manager/src/test/java/io/seldon/clustermanager/k8s/SeldonDeploymentDefaultingTest.java @@ -42,7 +42,7 @@ public void testDefaulting() throws IOException Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPortsCount()); Assert.assertEquals("http",mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPorts(0).getName()); Assert.assertEquals(Endpoint.EndpointType.REST_VALUE,mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getType().getNumber()); - Assert.assertEquals("0.0.0.0",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); + Assert.assertEquals("test-deployment-fx-market-predictor-mean-classifier",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); } @Test @@ -60,6 +60,6 @@ public void testDefaultingGrpc() throws IOException Assert.assertEquals(1,mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPortsCount()); Assert.assertEquals("grpc",mlDep2.getSpec().getPredictors(0).getComponentSpecs(0).getSpec().getContainers(0).getPorts(0).getName()); Assert.assertEquals(Endpoint.EndpointType.GRPC_VALUE,mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getType().getNumber()); - Assert.assertEquals("0.0.0.0",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); + Assert.assertEquals("test-deployment-fx-market-predictor-mean-classifier",mlDep2.getSpec().getPredictors(0).getGraph().getEndpoint().getServiceHost()); } } diff --git a/notebooks/kubectl_demo_minikube.ipynb b/notebooks/kubectl_demo_minikube.ipynb index 899c1690bf..53abffcfe9 100644 --- a/notebooks/kubectl_demo_minikube.ipynb +++ b/notebooks/kubectl_demo_minikube.ipynb @@ -30,9 +30,26 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting local Kubernetes v1.9.4 cluster...\n", + "Starting VM...\n", + "Getting VM IP address...\n", + "Moving files into cluster...\n", + "Setting up certs...\n", + "Connecting to cluster...\n", + "Setting up kubeconfig...\n", + "Starting cluster components...\n", + "Kubectl is now configured to use the cluster.\n", + "Loading cached images from config file.\n" + ] + } + ], "source": [ "!minikube start --memory=5000 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=AlwaysAllow" ] @@ -46,9 +63,20 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "$HELM_HOME has been configured at /home/clive/.helm.\n", + "\n", + "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n", + "Happy Helming!\n" + ] + } + ], "source": [ "!helm init" ] @@ -62,9 +90,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "node \"minikube\" labeled\r\n" + ] + } + ], "source": [ "!kubectl label nodes `kubectl get nodes -o jsonpath='{.items[0].metadata.name}'` role=locust --overwrite" ] @@ -85,9 +121,43 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-core-crd\n", + "LAST DEPLOYED: Mon Jun 11 19:56:24 2018\n", + "NAMESPACE: default\n", + "STATUS: DEPLOYED\n", + "\n", + "RESOURCES:\n", + "==> v1/ConfigMap\n", + "NAME DATA AGE\n", + "seldon-spartakus-config 3 0s\n", + "\n", + "==> v1beta1/CustomResourceDefinition\n", + "NAME AGE\n", + "seldondeployments.machinelearning.seldon.io 0s\n", + "\n", + "==> v1beta1/Deployment\n", + "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", + "seldon-spartakus-volunteer 1 1 1 0 0s\n", + "\n", + "==> v1/Pod(related)\n", + "NAME READY STATUS RESTARTS AGE\n", + "seldon-spartakus-volunteer-5977c9fdb8-kr74m 0/1 ContainerCreating 0 0s\n", + "\n", + "\n", + "NOTES:\n", + "NOTES: TODO\n", + "\n", + "\n" + ] + } + ], "source": [ "!helm install ../helm-charts/seldon-core-crd --name seldon-core-crd \\\n", " --set usage_metrics.enabled=true \\\n", @@ -96,18 +166,61 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "namespace \"seldon\" created\r\n" + ] + } + ], "source": [ "!kubectl create namespace seldon" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-core\r\n", + "LAST DEPLOYED: Mon Jun 11 19:58:11 2018\r\n", + "NAMESPACE: seldon\r\n", + "STATUS: DEPLOYED\r\n", + "\r\n", + "RESOURCES:\r\n", + "==> v1beta1/Deployment\r\n", + "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\r\n", + "seldon-apiserver 1 1 1 0 0s\r\n", + "seldon-cluster-manager 1 1 1 0 0s\r\n", + "redis 1 1 1 0 0s\r\n", + "\r\n", + "==> v1/Service\r\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\r\n", + "seldon-apiserver NodePort 10.109.80.190 8080:31062/TCP,5000:31612/TCP 0s\r\n", + "redis ClusterIP 10.110.233.69 6379/TCP 0s\r\n", + "\r\n", + "==> v1/Pod(related)\r\n", + "NAME READY STATUS RESTARTS AGE\r\n", + "seldon-apiserver-6fc74fc494-5mknt 0/1 ContainerCreating 0 0s\r\n", + "seldon-cluster-manager-86c6cb7b95-4bmzn 0/1 ContainerCreating 0 0s\r\n", + "redis-df886d999-htsnf 0/1 ContainerCreating 0 0s\r\n", + "\r\n", + "\r\n", + "NOTES:\r\n", + "NOTES: TODO\r\n", + "\r\n", + "\r\n" + ] + } + ], "source": [ "!helm install ../helm-charts/seldon-core --name seldon-core --namespace seldon \\\n", " --set rbac.enabled=false" @@ -182,7 +295,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -336,9 +449,17 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "map[predictorStatus:[map[name:test-deployment-fx-market-predictor-0 replicas:1 replicasAvailable:1]]]" + ] + } + ], "source": [ "!kubectl get seldondeployments seldon-deployment-example -o jsonpath='{.status}' -n seldon" ] @@ -359,9 +480,34 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"access_token\":\"5d6cc055-2dce-4aff-9356-bb662f5151b0\",\"token_type\":\"bearer\",\"expires_in\":42707,\"scope\":\"read write\"}\n", + "{\n", + " \"meta\": {\n", + " \"puid\": \"f43rqjbdmhorbjfdsevuuocknc\",\n", + " \"tags\": {\n", + " },\n", + " \"routing\": {\n", + " \"random-ab-test\": 0\n", + " }\n", + " },\n", + " \"data\": {\n", + " \"names\": [\"proba\"],\n", + " \"tensor\": {\n", + " \"shape\": [2, 1],\n", + " \"values\": [0.05133579311531625, 0.12823373759251927]\n", + " }\n", + " }\n", + "}\n" + ] + } + ], "source": [ "rest_request()" ] @@ -375,9 +521,31 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"access_token\":\"c533fa2b-5da2-4745-ba55-f9e4a353810e\",\"token_type\":\"bearer\",\"expires_in\":43181,\"scope\":\"read write\"}\n", + "meta {\n", + " puid: \"cgagftb7u1so8bu0eaetm77763\"\n", + "}\n", + "data {\n", + " names: \"proba\"\n", + " tensor {\n", + " shape: 3\n", + " shape: 1\n", + " values: 0.12823373759251927\n", + " values: 0.39731466202150834\n", + " values: 0.8296760813561542\n", + " }\n", + "}\n", + "\n" + ] + } + ], "source": [ "grpc_request()" ] diff --git a/notebooks/resources/random_ab_test.json b/notebooks/resources/random_ab_test.json index 0a12a9dd3b..0b06c104e3 100644 --- a/notebooks/resources/random_ab_test.json +++ b/notebooks/resources/random_ab_test.json @@ -32,9 +32,14 @@ }], "terminationGracePeriodSeconds": 20 }}, - { - "spec":{ - "containers":[ + { + "metadata":{ + "labels":{ + "version":"v2" + } + }, + "spec":{ + "containers":[ { "image": "seldonio/mock_classifier:1.0", "imagePullPolicy": "IfNotPresent", From 07dd5123a54011fc08614e90ae30cc334671ff2b Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 15 Jun 2018 18:22:26 +0100 Subject: [PATCH 05/20] use k8s/java-client 1.0.0. Add labels to proto. modify canary example so works with istio --- cluster-manager/pom.xml | 2 +- .../k8s/SeldonDeploymentOperatorImpl.java | 37 ++++++++++++------- .../k8s/SeldonDeploymentUtils.java | 2 +- .../k8s/SeldonDeploymentWatcher.java | 2 +- .../resources/model_invalid_no_method.json | 2 +- .../src/test/resources/model_simple.json | 1 - .../src/test/resources/model_simple_grpc.json | 1 - .../templates/seldon-deployment-crd.json | 8 +++- notebooks/resources/model_with_canary.json | 5 ++- proto/seldon_deployment.proto | 1 + 10 files changed, 39 insertions(+), 22 deletions(-) diff --git a/cluster-manager/pom.xml b/cluster-manager/pom.xml index 03d2466e37..110918b5ce 100644 --- a/cluster-manager/pom.xml +++ b/cluster-manager/pom.xml @@ -81,7 +81,7 @@ io.kubernetes client-java - 1.0.0-beta2-SNAPSHOT + 1.0.0 compile diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index dfd5c51d47..c077f7f42f 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -250,7 +250,9 @@ private V1.Container updateContainer(V1.Container c,PredictiveUnit pu,int portNu } } else - throw new UnsupportedOperationException(String.format("Found container port already set with http or grpc label. This is not presently allowed. Found port {}",containerPort)); + { + //throw new UnsupportedOperationException(String.format("Found container port already set with http or grpc label. This is not presently allowed. Found port {}",containerPort)); + } // Add environment variable for the port used in case the model needs to access it final String ENV_PREDICTIVE_UNIT_SERVICE_PORT ="PREDICTIVE_UNIT_SERVICE_PORT"; @@ -366,7 +368,6 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) { mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getComponentSpecsBuilder(ptsIdx).getSpecBuilder().addContainers(cIdx, c2); updatePredictiveUnitBuilderByName(mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getGraphBuilder(),c2,containerServiceValue); } - System.out.println("pbIdx"+pbIdx+" ptsIdx "+ptsIdx); mlBuilder.getSpecBuilder().getPredictorsBuilder(pbIdx).getComponentSpecsBuilder(ptsIdx).setMetadata(metaBuilder); } } @@ -479,13 +480,24 @@ private void addServicePort(PredictiveUnit pu,String serviceName,ServiceSpec.Bui Endpoint e = pu.getEndpoint(); if (e.getServiceHost().equals(serviceName)) { - svcSpecBuilder.addPorts(ServicePort.newBuilder() - .setProtocol("TCP") - .setPort(e.getServicePort()) - .setTargetPort(IntOrString.newBuilder().setIntVal(e.getServicePort())) - //.setName("http") - ); - return; + switch(e.getType()) + { + case REST: + svcSpecBuilder.addPorts(ServicePort.newBuilder() + .setProtocol("TCP") + .setPort(e.getServicePort()) + .setTargetPort(IntOrString.newBuilder().setIntVal(e.getServicePort())) + .setName("http") + ); + return; + case GRPC: + svcSpecBuilder.addPorts(ServicePort.newBuilder() + .setProtocol("TCP") + .setPort(e.getServicePort()) + .setTargetPort(IntOrString.newBuilder().setIntVal(e.getServicePort())) + .setName("grpc") + ); + } } } for(int i=0;i podLabel : spec.getMetadata().getLabelsMap().entrySet()) - //{ - // depMetaBuilder.put - //} for(V1.Container c : spec.getSpec().getContainersList()) { diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentUtils.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentUtils.java index de963b5a3c..dc85d7064f 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentUtils.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentUtils.java @@ -36,7 +36,7 @@ public class SeldonDeploymentUtils { public static SeldonDeployment jsonToSeldonDeployment(String json) throws InvalidProtocolBufferException { SeldonDeployment.Builder mlBuilder = SeldonDeployment.newBuilder(); - JsonFormat.parser().ignoringUnknownFields() + JsonFormat.parser()//.ignoringUnknownFields() .usingTypeParser(IntOrString.getDescriptor().getFullName(), new IntOrStringUtils.IntOrStringParser()) .usingTypeParser(Quantity.getDescriptor().getFullName(), new QuantityUtils.QuantityParser()) .usingTypeParser(Time.getDescriptor().getFullName(), new TimeUtils.TimeParser()) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java index 383bd7f0a5..16a4d64f88 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentWatcher.java @@ -20,7 +20,7 @@ import java.text.SimpleDateFormat; import java.util.Date; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; diff --git a/cluster-manager/src/test/resources/model_invalid_no_method.json b/cluster-manager/src/test/resources/model_invalid_no_method.json index 4755ca8b93..86b72b4121 100644 --- a/cluster-manager/src/test/resources/model_invalid_no_method.json +++ b/cluster-manager/src/test/resources/model_invalid_no_method.json @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": [{ + "componentSpecs": [{ "spec": { "containers": [ { diff --git a/cluster-manager/src/test/resources/model_simple.json b/cluster-manager/src/test/resources/model_simple.json index e6b4979943..41e02fae82 100644 --- a/cluster-manager/src/test/resources/model_simple.json +++ b/cluster-manager/src/test/resources/model_simple.json @@ -40,7 +40,6 @@ "endpoint": { "type" : "REST" }, - "subtype": "MICROSERVICE", "type": "MODEL" }, "name": "fx-market-predictor", diff --git a/cluster-manager/src/test/resources/model_simple_grpc.json b/cluster-manager/src/test/resources/model_simple_grpc.json index 27d33e772b..96b906051f 100644 --- a/cluster-manager/src/test/resources/model_simple_grpc.json +++ b/cluster-manager/src/test/resources/model_simple_grpc.json @@ -40,7 +40,6 @@ "endpoint": { "type" : "GRPC" }, - "subtype": "MICROSERVICE", "type": "MODEL" }, "name": "fx-market-predictor", diff --git a/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json b/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json index 7f285d4e04..2e4a37d75b 100644 --- a/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json +++ b/helm-charts/seldon-core-crd/templates/seldon-deployment-crd.json @@ -232,8 +232,12 @@ }, "type": "array" }, - "componentSpec": - {{ include "podTemplateSpec" . }} + "componentSpecs": + { + "description": "List of pods belonging to the predictor", + "type" : "array", + "items": {{ include "podTemplateSpec" . }} + } } } } diff --git a/notebooks/resources/model_with_canary.json b/notebooks/resources/model_with_canary.json index 0faef4801b..47860d8e24 100644 --- a/notebooks/resources/model_with_canary.json +++ b/notebooks/resources/model_with_canary.json @@ -69,7 +69,10 @@ "type": "MODEL" }, "name": "fx-market-predictor-canary", - "replicas": 1 + "replicas": 1, + "labels":{ + "version":"v2" + } } ] } diff --git a/proto/seldon_deployment.proto b/proto/seldon_deployment.proto index a8c84894fb..d2ccc702fa 100644 --- a/proto/seldon_deployment.proto +++ b/proto/seldon_deployment.proto @@ -48,6 +48,7 @@ message PredictorSpec { optional int32 replicas = 4; // The number of replicas of the predictor to create. map annotations = 5; // Arbitrary annotations. optional k8s.io.api.core.v1.ResourceRequirements engineResources = 6; // Optional set of resources for the Seldon engine which is added to each Predictor graph to manage the request/response flow + map labels = 7; // labels to be attached to entry deplyment for this predictor } From 02ae96f10961bfc9d1217684a04498abc75a6cd1 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Fri, 15 Jun 2018 19:14:56 +0100 Subject: [PATCH 06/20] change seldon service name function --- .../k8s/SeldonDeploymentOperator.java | 3 ++- .../k8s/SeldonDeploymentOperatorImpl.java | 26 +++++++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java index 4fdc8a5f44..ea12b7486a 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperator.java @@ -16,6 +16,7 @@ package io.seldon.clustermanager.k8s; import io.seldon.clustermanager.k8s.SeldonDeploymentOperatorImpl.DeploymentResources; +import io.seldon.protos.DeploymentProtos.PredictorSpec; import io.seldon.protos.DeploymentProtos.SeldonDeployment; public interface SeldonDeploymentOperator { @@ -23,6 +24,6 @@ public interface SeldonDeploymentOperator { public SeldonDeployment defaulting(SeldonDeployment mlDep); public void validate(SeldonDeployment mlDep) throws SeldonDeploymentException; public DeploymentResources createResources(SeldonDeployment mlDep) throws SeldonDeploymentException; - public String getKubernetesDeploymentName(String depName,String predictorName,int podTemplateIdx); + public String getSeldonServiceName(SeldonDeployment dep,PredictorSpec pred,String key); } diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index c077f7f42f..5e1548bff5 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -315,19 +315,20 @@ private void updatePredictiveUnitBuilderByName(PredictiveUnit.Builder puBuilder, } } + /* private String getPredictorServiceNameValue(SeldonDeployment mlDep,String predictorName,String containerName) { return mlDep.getSpec().getName()+"-"+predictorName+"-"+containerName; } - + */ private String getPredictorServiceNameKey(String containerName) { return LABEL_SELDON_APP+"-"+containerName; } @Override - public String getKubernetesDeploymentName(String deploymentName,String predictorName,int podTemplateIdx) { - return deploymentName + "-" + predictorName+"-"+podTemplateIdx; + public String getSeldonServiceName(SeldonDeployment dep,PredictorSpec pred,String key) { + return dep.getSpec().getName() + "-" + pred.getName()+"-"+key; } @@ -352,7 +353,7 @@ public SeldonDeployment defaulting(SeldonDeployment mlDep) { { V1.Container c = spec.getSpec().getContainers(cIdx); String containerServiceKey = getPredictorServiceNameKey(c.getName()); - String containerServiceValue = getPredictorServiceNameValue(mlDep, p.getName(), c.getName()); + String containerServiceValue = getSeldonServiceName(mlDep, p, c.getName()); metaBuilder.putLabels(containerServiceKey, containerServiceValue); int portNum; @@ -527,18 +528,21 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon .putAnnotations("prometheus.io/path", "/prometheus") .putAnnotations("prometheus.io/port",""+clusterManagerProperites.getEngineContainerPort()) .putAnnotations("prometheus.io/scrape", "true"); - String depName = getKubernetesDeploymentName(mlDep.getSpec().getName(),p.getName(),0); + String depName = getSeldonServiceName(mlDep,p,"svc-orch"); ObjectMeta.Builder depMetaBuilder = ObjectMeta.newBuilder() .setName(depName) .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP, serviceLabel) .putLabels(Constants.LABEL_SELDON_ID, mlDep.getSpec().getName()) .putLabels("app", depName) - .putLabels("version", "v1") //FIXME + .putLabels("version", "v1") .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_KEY, SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_VAL) .addOwnerReferences(ownerRef); - depMetaBuilder.putAllLabels(p.getLabelsMap()); + + // Add default version number then overwrite with any labels podSpecBuilder.getMetadataBuilder().putLabels("version", "v1"); + depMetaBuilder.putAllLabels(p.getLabelsMap()); podSpecBuilder.getMetadataBuilder().putAllLabels(p.getLabelsMap()); + Deployment deployment = V1beta1Extensions.Deployment.newBuilder() .setMetadata(depMetaBuilder) .setSpec(DeploymentSpec.newBuilder() @@ -554,23 +558,23 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon for(int ptsIdx=0;ptsIdx Date: Sat, 16 Jun 2018 11:16:49 +0100 Subject: [PATCH 07/20] ensire correct labels for istio telemetry --- .../k8s/SeldonDeploymentOperatorImpl.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index 5e1548bff5..1feb3d58cb 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -497,7 +497,8 @@ private void addServicePort(PredictiveUnit pu,String serviceName,ServiceSpec.Bui .setPort(e.getServicePort()) .setTargetPort(IntOrString.newBuilder().setIntVal(e.getServicePort())) .setName("grpc") - ); + ); + return; } } } @@ -523,12 +524,14 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon podSpecBuilder.getSpecBuilder() .addContainers(createEngineContainer(mlDep,p)) .setTerminationGracePeriodSeconds(20); + String depName = getSeldonServiceName(mlDep,p,"svc-orch"); podSpecBuilder.getMetadataBuilder() .putLabels(LABEL_SELDON_APP, mlDep.getSpec().getName()) + .putLabels("app", depName) .putAnnotations("prometheus.io/path", "/prometheus") .putAnnotations("prometheus.io/port",""+clusterManagerProperites.getEngineContainerPort()) .putAnnotations("prometheus.io/scrape", "true"); - String depName = getSeldonServiceName(mlDep,p,"svc-orch"); + ObjectMeta.Builder depMetaBuilder = ObjectMeta.newBuilder() .setName(depName) .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_APP, serviceLabel) @@ -568,8 +571,10 @@ public DeploymentResources createResources(SeldonDeployment mlDep) throws Seldon .putLabels(SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_KEY, SeldonDeploymentOperatorImpl.LABEL_SELDON_TYPE_VAL) .addOwnerReferences(ownerRef); - //Overwrite labels + // Add default version number then overwrite with any labels + podSpecBuilder.getMetadataBuilder().putLabels("version", "v1"); depMetaBuilder.putAllLabels(spec.getMetadata().getLabelsMap()); + podSpecBuilder.getMetadataBuilder().putAllLabels(spec.getMetadata().getLabelsMap()); for(V1.Container c : spec.getSpec().getContainersList()) { From 0afa06a97bc9ea995755cc195a0e16a3cff4c1ba Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 16 Jun 2018 11:31:31 +0100 Subject: [PATCH 08/20] update to v1alpha2 in operator tests --- cluster-manager/src/test/resources/model_invalid_graph.json | 2 +- cluster-manager/src/test/resources/model_invalid_no_method.json | 2 +- cluster-manager/src/test/resources/model_simple.json | 2 +- cluster-manager/src/test/resources/model_simple_grpc.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-manager/src/test/resources/model_invalid_graph.json b/cluster-manager/src/test/resources/model_invalid_graph.json index c263e39ed0..dd3b85acee 100644 --- a/cluster-manager/src/test/resources/model_invalid_graph.json +++ b/cluster-manager/src/test/resources/model_invalid_graph.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { diff --git a/cluster-manager/src/test/resources/model_invalid_no_method.json b/cluster-manager/src/test/resources/model_invalid_no_method.json index 86b72b4121..4580d070a3 100644 --- a/cluster-manager/src/test/resources/model_invalid_no_method.json +++ b/cluster-manager/src/test/resources/model_invalid_no_method.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { diff --git a/cluster-manager/src/test/resources/model_simple.json b/cluster-manager/src/test/resources/model_simple.json index 41e02fae82..7bd2ef611f 100644 --- a/cluster-manager/src/test/resources/model_simple.json +++ b/cluster-manager/src/test/resources/model_simple.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { diff --git a/cluster-manager/src/test/resources/model_simple_grpc.json b/cluster-manager/src/test/resources/model_simple_grpc.json index 96b906051f..58d7f2d485 100644 --- a/cluster-manager/src/test/resources/model_simple_grpc.json +++ b/cluster-manager/src/test/resources/model_simple_grpc.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { From e08dd739a5d0942e7b9425de03ffd01b4ce1e4f5 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 17 Jun 2018 18:40:26 +0100 Subject: [PATCH 09/20] fix typos in proto in field numbers --- proto/seldon_deployment.proto | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/proto/seldon_deployment.proto b/proto/seldon_deployment.proto index d2ccc702fa..24aa1a4ae7 100644 --- a/proto/seldon_deployment.proto +++ b/proto/seldon_deployment.proto @@ -36,9 +36,9 @@ message PredictorStatus { message DeploymentSpec { optional string name = 1; // A unique name within the namespace. repeated PredictorSpec predictors = 2; // A list of 1 or more predictors describing runtime machine learning deployment graphs. - optional string oauth_key = 6; // The oauth key for external users to use this deployment via an API. - optional string oauth_secret = 7; // The oauth secret for external users to use this deployment via an API. - map annotations = 8; // Arbitrary annotations. + optional string oauth_key = 3; // The oauth key for external users to use this deployment via an API. + optional string oauth_secret = 4; // The oauth secret for external users to use this deployment via an API. + map annotations = 5; // Arbitrary annotations. } message PredictorSpec { From 3d61012c28e1106ca263630c38e9880597d53004 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 17 Jun 2018 18:40:48 +0100 Subject: [PATCH 10/20] update ksonnset CRD --- seldon-core/seldon-core/crd.libsonnet | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/seldon-core/seldon-core/crd.libsonnet b/seldon-core/seldon-core/crd.libsonnet index 184f4ca1d3..b0019f9d83 100644 --- a/seldon-core/seldon-core/crd.libsonnet +++ b/seldon-core/seldon-core/crd.libsonnet @@ -108,7 +108,7 @@ local podTemplateValidation = import 'json/pod-template-spec-validation.json'; "ROUTE", "AGGREGATE", "SEND_FEEDBACK", -], + ], type: "string", }, }, @@ -166,7 +166,7 @@ local podTemplateValidation = import 'json/pod-template-spec-validation.json'; "ROUTE", "AGGREGATE", "SEND_FEEDBACK", -], + ], type: "string", }, }, @@ -224,7 +224,7 @@ local podTemplateValidation = import 'json/pod-template-spec-validation.json'; "ROUTE", "AGGREGATE", "SEND_FEEDBACK", -], + ], type: "string", }, }, @@ -240,15 +240,19 @@ local podTemplateValidation = import 'json/pod-template-spec-validation.json'; }, type: "array", }, - componentSpec: podTemplateValidation, - + componentSpecs: + { + description: "List of pods belonging to the predictor", + "type" : "array", + items: podTemplateValidation, + }, }, }, }, }, }, - version: "v1alpha1", + version: "v1alpha2", }, }, -} +}, From 616b4f7cc397606184a8fda74a39311929f0936f Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 17 Jun 2018 18:41:21 +0100 Subject: [PATCH 11/20] initial update to docs for v1alpha2 --- docs/getting_started/minikube.md | 6 +++--- docs/reference/seldon-deployment.md | 22 ++++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/getting_started/minikube.md b/docs/getting_started/minikube.md index 52940ef7f2..3b931ad4ed 100644 --- a/docs/getting_started/minikube.md +++ b/docs/getting_started/minikube.md @@ -94,7 +94,7 @@ The docker image version of your model is deployed through a json configuration ```json { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -112,7 +112,7 @@ The docker image version of your model is deployed through a json configuration "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -128,7 +128,7 @@ The docker image version of your model is deployed through a json configuration ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "sklearn-iris-classifier", diff --git a/docs/reference/seldon-deployment.md b/docs/reference/seldon-deployment.md index 326ca907fc..0c828581c5 100644 --- a/docs/reference/seldon-deployment.md +++ b/docs/reference/seldon-deployment.md @@ -53,17 +53,19 @@ message PredictorStatus { message DeploymentSpec { optional string name = 1; // A unique name within the namespace. repeated PredictorSpec predictors = 2; // A list of 1 or more predictors describing runtime machine learning deployment graphs. - optional string oauth_key = 6; // The oauth key for external users to use this deployment via an API. - optional string oauth_secret = 7; // The oauth secret for external users to use this deployment via an API. - map annotations = 8; // Arbitrary annotations. + optional string oauth_key = 3; // The oauth key for external users to use this deployment via an API. + optional string oauth_secret = 4; // The oauth secret for external users to use this deployment via an API. + map annotations = 5; // Arbitrary annotations. } message PredictorSpec { required string name = 1; // A unique name not used by any other predictor in the deployment. required PredictiveUnit graph = 2; // A graph describing how the predictive units are connected together. - required k8s.io.api.core.v1.PodTemplateSpec componentSpec = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. + repeated k8s.io.api.core.v1.PodTemplateSpec componentSpecs = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. Can be split over 1 or more PodTemplateSpecs. optional int32 replicas = 4; // The number of replicas of the predictor to create. map annotations = 5; // Arbitrary annotations. + optional k8s.io.api.core.v1.ResourceRequirements engineResources = 6; // Optional set of resources for the Seldon engine which is added to each Predictor graph to manage the request/response flow + map labels = 7; // labels to be attached to entry deplyment for this predictor } @@ -149,7 +151,7 @@ message Parameter { ```json { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -167,13 +169,13 @@ message Parameter { "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { - "image": "seldonio/mean_classifier:0.6", + "image": "seldonio/mock_classifier:1.0", "imagePullPolicy": "IfNotPresent", - "name": "mean-classifier", + "name": "classifier", "resources": { "requests": { "memory": "1Mi" @@ -183,10 +185,10 @@ message Parameter { ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], - "name": "mean-classifier", + "name": "classifier", "endpoint": { "type" : "REST" }, From 7d77bef2b165c563d4c1afc6d376f5fa77468237 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 17 Jun 2018 18:41:44 +0100 Subject: [PATCH 12/20] remove k8s client java local install from Makefile.ci --- cluster-manager/Makefile.ci | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cluster-manager/Makefile.ci b/cluster-manager/Makefile.ci index f84c7b3f41..3a5f9804ba 100644 --- a/cluster-manager/Makefile.ci +++ b/cluster-manager/Makefile.ci @@ -5,7 +5,7 @@ VERSION_FILE=target/version.txt build: clean build_image -build_jar: update_proto k8s_java_client +build_jar: update_proto mvn clean package -B write_version: build_jar @@ -40,8 +40,4 @@ update_proto: download_protos cp -vr ../proto/k8s/k8s.io src/main/proto cp -v ../proto/k8s/v1.proto src/main/proto -k8s_java_client: - git clone -b updated_protos https://github.com/cliveseldon/java.git java_client && \ - cd java_client && \ - mvn install From ffdd2ca008971870dc374888a1d825068bbe8bae Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sun, 17 Jun 2018 18:49:33 +0100 Subject: [PATCH 13/20] update example json resource files for v1alpha2 --- examples/models/deep_mnist/deep_mnist.json | 6 +++--- examples/models/h2o_example/h2o_badloans_deployment.json | 6 +++--- examples/models/h2o_mojo/h2o_deployment.json | 6 +++--- examples/models/keras_mnist/keras_mnist_deployment.json | 6 +++--- examples/models/pyspark_pmml/mnist_deployment.json | 6 +++--- examples/models/r_iris/r_iris_deployment.json | 6 +++--- examples/models/r_mnist/r_mnist_deployment.json | 6 +++--- examples/models/sklearn_iris/sklearn_iris_deployment.json | 6 +++--- .../models/sklearn_iris_docker/sklearn_iris_deployment.json | 6 +++--- examples/models/templates/deployment.json | 6 +++--- 10 files changed, 30 insertions(+), 30 deletions(-) diff --git a/examples/models/deep_mnist/deep_mnist.json b/examples/models/deep_mnist/deep_mnist.json index 44710cc199..350a4c85bc 100644 --- a/examples/models/deep_mnist/deep_mnist.json +++ b/examples/models/deep_mnist/deep_mnist.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "classifier", diff --git a/examples/models/h2o_example/h2o_badloans_deployment.json b/examples/models/h2o_example/h2o_badloans_deployment.json index 685697a81b..077aa980b1 100644 --- a/examples/models/h2o_example/h2o_badloans_deployment.json +++ b/examples/models/h2o_example/h2o_badloans_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "h2o-bad-loans-classifier", diff --git a/examples/models/h2o_mojo/h2o_deployment.json b/examples/models/h2o_mojo/h2o_deployment.json index f31d73f8fb..98096b3ee6 100644 --- a/examples/models/h2o_mojo/h2o_deployment.json +++ b/examples/models/h2o_mojo/h2o_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "prostate-classifier", diff --git a/examples/models/keras_mnist/keras_mnist_deployment.json b/examples/models/keras_mnist/keras_mnist_deployment.json index 854171444f..23fc266398 100644 --- a/examples/models/keras_mnist/keras_mnist_deployment.json +++ b/examples/models/keras_mnist/keras_mnist_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "keras-mnist-classifier", diff --git a/examples/models/pyspark_pmml/mnist_deployment.json b/examples/models/pyspark_pmml/mnist_deployment.json index 79586c4482..efc9998b24 100644 --- a/examples/models/pyspark_pmml/mnist_deployment.json +++ b/examples/models/pyspark_pmml/mnist_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "mnist-classifier", diff --git a/examples/models/r_iris/r_iris_deployment.json b/examples/models/r_iris/r_iris_deployment.json index e4183bce90..28c6798173 100644 --- a/examples/models/r_iris/r_iris_deployment.json +++ b/examples/models/r_iris/r_iris_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "r-iris-classifier", diff --git a/examples/models/r_mnist/r_mnist_deployment.json b/examples/models/r_mnist/r_mnist_deployment.json index 8f5e4e646b..52f6fa6346 100644 --- a/examples/models/r_mnist/r_mnist_deployment.json +++ b/examples/models/r_mnist/r_mnist_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "r-mnist-classifier", diff --git a/examples/models/sklearn_iris/sklearn_iris_deployment.json b/examples/models/sklearn_iris/sklearn_iris_deployment.json index 686e417a92..dc793a12d9 100644 --- a/examples/models/sklearn_iris/sklearn_iris_deployment.json +++ b/examples/models/sklearn_iris/sklearn_iris_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "sklearn-iris-classifier", diff --git a/examples/models/sklearn_iris_docker/sklearn_iris_deployment.json b/examples/models/sklearn_iris_docker/sklearn_iris_deployment.json index 08d930ef31..df3204d1dd 100644 --- a/examples/models/sklearn_iris_docker/sklearn_iris_deployment.json +++ b/examples/models/sklearn_iris_docker/sklearn_iris_deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "oauth-secret", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": 20 } - }, + }], "graph": { "children": [], "name": "sklearn-iris-classifier", diff --git a/examples/models/templates/deployment.json b/examples/models/templates/deployment.json index c1df8668ad..11ca62842a 100644 --- a/examples/models/templates/deployment.json +++ b/examples/models/templates/deployment.json @@ -1,5 +1,5 @@ { - "apiVersion": "machinelearning.seldon.io/v1alpha1", + "apiVersion": "machinelearning.seldon.io/v1alpha2", "kind": "SeldonDeployment", "metadata": { "labels": { @@ -17,7 +17,7 @@ "oauth_secret": "", "predictors": [ { - "componentSpec": { + "componentSpecs": [{ "spec": { "containers": [ { @@ -33,7 +33,7 @@ ], "terminationGracePeriodSeconds": <> } - }, + }], "graph": { "children": [], "name": "", From af8d087936d852c3a8847d9f434d3ae9c3edc51c Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Tue, 19 Jun 2018 19:07:04 +0100 Subject: [PATCH 14/20] initial istio canary example --- .../service/InternalPredictionService.java | 3 +- examples/istio/canary_update/canary.ipynb | 670 ++++++++++++++++++ .../istio/canary_update/istio_canary_v1.yaml | 30 + .../istio/canary_update/istio_canary_v2.yaml | 35 + .../istio/canary_update/istio_canary_v3.yaml | 30 + examples/istio/canary_update/mnist_v1.json | 52 ++ examples/istio/canary_update/mnist_v2.json | 85 +++ examples/istio/canary_update/utils.py | 83 +++ 8 files changed, 987 insertions(+), 1 deletion(-) create mode 100644 examples/istio/canary_update/canary.ipynb create mode 100644 examples/istio/canary_update/istio_canary_v1.yaml create mode 100644 examples/istio/canary_update/istio_canary_v2.yaml create mode 100644 examples/istio/canary_update/istio_canary_v3.yaml create mode 100644 examples/istio/canary_update/mnist_v1.json create mode 100644 examples/istio/canary_update/mnist_v2.json create mode 100644 examples/istio/canary_update/utils.py diff --git a/api-frontend/src/main/java/io/seldon/apife/service/InternalPredictionService.java b/api-frontend/src/main/java/io/seldon/apife/service/InternalPredictionService.java index ccc7b186ac..009d7ce380 100644 --- a/api-frontend/src/main/java/io/seldon/apife/service/InternalPredictionService.java +++ b/api-frontend/src/main/java/io/seldon/apife/service/InternalPredictionService.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.util.concurrent.TimeUnit; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; @@ -56,7 +57,7 @@ public class InternalPredictionService { @Autowired public InternalPredictionService(AppProperties appProperties){ this.appProperties = appProperties; - connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager = new PoolingHttpClientConnectionManager(10,TimeUnit.SECONDS); connectionManager.setMaxTotal(150); connectionManager.setDefaultMaxPerRoute(150); diff --git a/examples/istio/canary_update/canary.ipynb b/examples/istio/canary_update/canary.ipynb new file mode 100644 index 0000000000..45e4cb0ae2 --- /dev/null +++ b/examples/istio/canary_update/canary.ipynb @@ -0,0 +1,670 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "env: ISTIO_HOME=/home/clive/work/istio/istio-0.8.0\n" + ] + } + ], + "source": [ + "%env ISTIO_HOME=/home/clive/work/istio/istio-0.8.0" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "serviceaccount \"tiller\" created\r\n", + "clusterrolebinding \"tiller\" created\r\n" + ] + } + ], + "source": [ + "!kubectl create -f ${ISTIO_HOME}/install/kubernetes/helm/helm-service-account.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "$HELM_HOME has been configured at /home/clive/.helm.\n", + "\n", + "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n", + "Happy Helming!\n" + ] + } + ], + "source": [ + "!helm init --service-account tiller" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: istio\n", + "LAST DEPLOYED: Tue Jun 19 18:45:17 2018\n", + "NAMESPACE: istio-system\n", + "STATUS: DEPLOYED\n", + "\n", + "RESOURCES:\n", + "==> v1beta1/CustomResourceDefinition\n", + "NAME AGE\n", + "deniers.config.istio.io 45s\n", + "rbacs.config.istio.io 45s\n", + "prometheuses.config.istio.io 45s\n", + "servicecontrolreports.config.istio.io 45s\n", + "logentries.config.istio.io 45s\n", + "listentries.config.istio.io 45s\n", + "kuberneteses.config.istio.io 45s\n", + "fluentds.config.istio.io 45s\n", + "kubernetesenvs.config.istio.io 45s\n", + "stdios.config.istio.io 45s\n", + "circonuses.config.istio.io 45s\n", + "listcheckers.config.istio.io 45s\n", + "solarwindses.config.istio.io 45s\n", + "apikeys.config.istio.io 45s\n", + "rules.config.istio.io 45s\n", + "quotas.config.istio.io 45s\n", + "reportnothings.config.istio.io 45s\n", + "tracespans.config.istio.io 45s\n", + "serviceroles.config.istio.io 45s\n", + "servicerolebindings.config.istio.io 45s\n", + "noops.config.istio.io 45s\n", + "opas.config.istio.io 45s\n", + "statsds.config.istio.io 45s\n", + "checknothings.config.istio.io 45s\n", + "attributemanifests.config.istio.io 45s\n", + "memquotas.config.istio.io 45s\n", + "stackdrivers.config.istio.io 45s\n", + "authorizations.config.istio.io 45s\n", + "metrics.config.istio.io 45s\n", + "servicecontrols.config.istio.io 45s\n", + "routerules.config.istio.io 45s\n", + "quotaspecbindings.config.istio.io 45s\n", + "destinationpolicies.config.istio.io 45s\n", + "egressrules.config.istio.io 45s\n", + "httpapispecs.config.istio.io 45s\n", + "quotaspecs.config.istio.io 45s\n", + "httpapispecbindings.config.istio.io 45s\n", + "gateways.networking.istio.io 45s\n", + "destinationrules.networking.istio.io 45s\n", + "virtualservices.networking.istio.io 45s\n", + "serviceentries.networking.istio.io 45s\n", + "policies.authentication.istio.io 45s\n", + "\n", + "==> v1beta1/RoleBinding\n", + "NAME AGE\n", + "istio-cleanup-old-ca-istio-system 45s\n", + "\n", + "==> v2beta1/HorizontalPodAutoscaler\n", + "NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE\n", + "istio-egressgateway Deployment/istio-egressgateway / 80% 1 1 1 45s\n", + "istio-ingress Deployment/istio-ingress / 80% 1 1 1 45s\n", + "istio-ingressgateway Deployment/istio-ingressgateway / 80% 1 1 1 45s\n", + "\n", + "==> v1beta1/MutatingWebhookConfiguration\n", + "NAME AGE\n", + "istio-sidecar-injector 45s\n", + "\n", + "==> v1/Pod(related)\n", + "NAME READY STATUS RESTARTS AGE\n", + "istio-egressgateway-58d98d898c-ftrxw 1/1 Running 0 45s\n", + "istio-ingress-6fb78f687f-m4c47 1/1 Running 0 45s\n", + "istio-ingressgateway-6bc7c7c4bc-km9xn 0/1 ContainerCreating 0 45s\n", + "istio-telemetry-54b5bf4847-ndp9d 2/2 Running 0 45s\n", + "istio-policy-5c7fbb4b9f-mdwcl 0/2 ContainerCreating 0 45s\n", + "istio-statsd-prom-bridge-6dbb7dcc7f-qxhxg 0/1 ContainerCreating 0 45s\n", + "istio-pilot-6c5c6b586c-xgp4q 0/2 ContainerCreating 0 45s\n", + "prometheus-586d95b8d9-6wfbj 0/1 ContainerCreating 0 45s\n", + "istio-citadel-ff5696f6f-78mt5 0/1 ContainerCreating 0 44s\n", + "istio-sidecar-injector-dbd67c88d-glfbf 0/1 ContainerCreating 0 44s\n", + "\n", + "==> v1/ConfigMap\n", + "NAME DATA AGE\n", + "istio-statsd-prom-bridge 1 45s\n", + "istio-mixer-custom-resources 1 45s\n", + "prometheus 1 45s\n", + "istio 1 45s\n", + "istio-sidecar-injector 1 45s\n", + "\n", + "==> v1/ServiceAccount\n", + "NAME SECRETS AGE\n", + "istio-egressgateway-service-account 1 45s\n", + "istio-ingress-service-account 1 45s\n", + "istio-ingressgateway-service-account 1 45s\n", + "istio-mixer-post-install-account 1 45s\n", + "istio-mixer-service-account 1 45s\n", + "istio-pilot-service-account 1 45s\n", + "prometheus 1 45s\n", + "istio-cleanup-old-ca-service-account 1 45s\n", + "istio-citadel-service-account 1 45s\n", + "istio-sidecar-injector-service-account 1 45s\n", + "\n", + "==> v1beta1/ClusterRole\n", + "NAME AGE\n", + "istio-ingress-istio-system 45s\n", + "istio-mixer-istio-system 45s\n", + "istio-mixer-post-install-istio-system 45s\n", + "istio-pilot-istio-system 45s\n", + "prometheus-istio-system 45s\n", + "istio-citadel-istio-system 45s\n", + "istio-sidecar-injector-istio-system 45s\n", + "\n", + "==> v1beta1/ClusterRoleBinding\n", + "NAME AGE\n", + "istio-ingress-istio-system 45s\n", + "istio-mixer-admin-role-binding-istio-system 45s\n", + "istio-mixer-post-install-role-binding-istio-system 45s\n", + "istio-pilot-istio-system 45s\n", + "prometheus-istio-system 45s\n", + "istio-citadel-istio-system 45s\n", + "istio-sidecar-injector-admin-role-binding-istio-system 45s\n", + "\n", + "==> v1beta1/Role\n", + "NAME AGE\n", + "istio-cleanup-old-ca-istio-system 45s\n", + "\n", + "==> v1/Service\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "istio-egressgateway ClusterIP 10.102.213.53 80/TCP,443/TCP 45s\n", + "istio-ingress LoadBalancer 10.106.70.233 80:32000/TCP,443:31186/TCP 45s\n", + "istio-ingressgateway LoadBalancer 10.106.229.108 80:31380/TCP,443:31390/TCP,31400:31400/TCP 45s\n", + "istio-policy ClusterIP 10.111.134.24 9091/TCP,15004/TCP,9093/TCP 45s\n", + "istio-telemetry ClusterIP 10.98.139.75 9091/TCP,15004/TCP,9093/TCP,42422/TCP 45s\n", + "istio-statsd-prom-bridge ClusterIP 10.101.127.9 9102/TCP,9125/UDP 45s\n", + "istio-pilot ClusterIP 10.108.21.76 15003/TCP,15005/TCP,15007/TCP,15010/TCP,15011/TCP,8080/TCP,9093/TCP 45s\n", + "prometheus ClusterIP 10.109.91.39 9090/TCP 45s\n", + "istio-citadel ClusterIP 10.98.205.164 8060/TCP,9093/TCP 45s\n", + "istio-sidecar-injector ClusterIP 10.103.0.45 443/TCP 45s\n", + "\n", + "==> v1beta1/Deployment\n", + "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", + "istio-egressgateway 1 1 1 1 45s\n", + "istio-ingress 1 1 1 1 45s\n", + "istio-ingressgateway 1 1 1 0 45s\n", + "istio-telemetry 1 1 1 1 45s\n", + "istio-policy 1 1 1 0 45s\n", + "istio-statsd-prom-bridge 1 1 1 0 45s\n", + "istio-pilot 1 1 1 0 45s\n", + "prometheus 1 1 1 0 45s\n", + "istio-citadel 1 1 1 0 45s\n", + "istio-sidecar-injector 1 1 1 0 45s\n", + "\n", + "\n" + ] + } + ], + "source": [ + "!helm install ${ISTIO_HOME}/install/kubernetes/helm/istio --name istio --namespace istio-system \\\n", + " --set global.proxy.includeIPRanges=\"10.0.0.1/8\"" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "service \"grafana\" created\n", + "deployment \"grafana\" created\n", + "serviceaccount \"grafana\" created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ${ISTIO_HOME}/install/kubernetes/addons/grafana.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000\n", + "```\n", + "http://localhost:3000/dashboard/db/istio-dashboard" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl create namespace seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl label namespace seldon istio-injection=enabled" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "clusterrole \"ambassador\" created\n", + "serviceaccount \"ambassador\" created\n", + "clusterrolebinding \"ambassador\" created\n", + "service \"ambassador\" created\n", + "service \"ambassador-admin\" created\n", + "deployment \"ambassador\" created\n" + ] + } + ], + "source": [ + "!kubectl apply -f ../../../notebooks/resources/ambassador-rbac.yaml -n seldon" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: seldon-core-analytics\n", + "LAST DEPLOYED: Tue Jun 19 18:49:33 2018\n", + "NAMESPACE: seldon\n", + "STATUS: DEPLOYED\n", + "\n", + "RESOURCES:\n", + "==> v1/Service\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "alertmanager ClusterIP 10.97.10.96 80/TCP 1s\n", + "grafana-prom NodePort 10.101.14.81 80:31040/TCP 1s\n", + "prometheus-node-exporter ClusterIP None 9100/TCP 1s\n", + "prometheus-seldon ClusterIP 10.103.146.190 80/TCP 1s\n", + "\n", + "==> v1/Pod(related)\n", + "NAME READY STATUS RESTARTS AGE\n", + "grafana-prom-import-dashboards-mp82k 0/2 Init:0/1 0 1s\n", + "alertmanager-deployment-7fbfdfdfb6-tr8fj 0/2 Init:0/1 0 1s\n", + "grafana-prom-deployment-7b45fb85d4-htvvh 0/2 Init:0/1 0 1s\n", + "prometheus-node-exporter-wt5hk 0/1 Pending 0 1s\n", + "prometheus-deployment-cbfd78cc7-qwxx9 0/2 Pending 0 1s\n", + "\n", + "==> v1/ServiceAccount\n", + "NAME SECRETS AGE\n", + "prometheus 1 1s\n", + "\n", + "==> v1beta1/ClusterRole\n", + "NAME AGE\n", + "prometheus 1s\n", + "\n", + "==> v1beta1/ClusterRoleBinding\n", + "NAME AGE\n", + "prometheus 1s\n", + "\n", + "==> v1/Job\n", + "NAME DESIRED SUCCESSFUL AGE\n", + "grafana-prom-import-dashboards 1 0 1s\n", + "\n", + "==> v1beta1/Deployment\n", + "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", + "alertmanager-deployment 1 1 1 0 1s\n", + "grafana-prom-deployment 1 1 1 0 1s\n", + "prometheus-deployment 1 1 1 0 1s\n", + "\n", + "==> v1/Secret\n", + "NAME TYPE DATA AGE\n", + "grafana-prom-secret Opaque 1 1s\n", + "\n", + "==> v1/ConfigMap\n", + "NAME DATA AGE\n", + "alertmanager-server-conf 1 1s\n", + "grafana-import-dashboards 5 1s\n", + "prometheus-rules 4 1s\n", + "prometheus-server-conf 1 1s\n", + "\n", + "==> v1beta1/DaemonSet\n", + "NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE\n", + "prometheus-node-exporter 1 1 0 1 0 1s\n", + "\n", + "\n", + "NOTES:\n", + "NOTES: TODO\n", + "\n", + "\n" + ] + } + ], + "source": [ + "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", + " --set grafana_prom_admin_password=password \\\n", + " --set persistence.enabled=false \\\n", + " --namespace seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```\n", + "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3001:3000\n", + "```\n", + "\n", + "http://localhost:3001/dashboard/db/prediction-analytics?refresh=5s&orgId=1" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Context \"minikube\" modified.\r\n" + ] + } + ], + "source": [ + "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "!mkdir -p proto\n", + "!cp ../../../proto/prediction.proto ./proto\n", + "!python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", + "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", + "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", + "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" + ] + } + ], + "source": [ + "%matplotlib inline\n", + "import utils\n", + "mnist = utils.download_mnist()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl create -f mnist_v1.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl get pods" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADbtJREFUeJzt3W+sVPWdx/HPd1lIVDCBMAgB3FuqMf4Jhc1INnFj2BiJ3TS5FlNTNIhJLT4oiU36YBVNypNNzLpt1wcrerve9GpaaQ1V0Bi3xBi1yaZhJFro4hYkl8JycxlEU9AHKHz3wT10r3jnN+Occ+bM5ft+JWRmzvf8+Trxc8/M/GbOz9xdAOL5q6obAFANwg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+IKi/7uXB5s+f7wMDA708JBDK6OioTpw4YZ2smyv8ZnabpMclzZD0H+7+aGr9gYEBNRqNPIcEkFCv1ztet+uX/WY2Q9K/S/q6pOskrTOz67rdH4DeyvOef5Wkg+5+yN3PSNomabCYtgCULU/4F0s6Munx0WzZ55jZRjNrmFmj2WzmOByAIuUJ/1QfKnzh98HuPuTudXev12q1HIcDUKQ84T8qaemkx0skHcvXDoBeyRP+3ZKuNrOvmNksSd+WtLOYtgCUreuhPnf/zMw2SfpPTQz1Dbv7HwrrDECpco3zu/srkl4pqBcAPcTXe4GgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwgq1yy9ZjYq6ZSks5I+c/d6EU0BknTmzJlk/YknnkjWn3rqqZa1/fv3d9XTxSRX+DP/4O4nCtgPgB7iZT8QVN7wu6TfmNnbZraxiIYA9Ebel/03ufsxM1sgaZeZvefub05eIfujsFGSrrzyypyHA1CUXGd+dz+W3R6X9IKkVVOsM+TudXev12q1PIcDUKCuw29ml5nZnPP3Ja2RtK+oxgCUK8/L/iskvWBm5/fzC3d/tZCuAJSu6/C7+yFJXyuwl7A++eSTZP35559P1u+4446WtdmzZ3fVUz944IEHkvUnn3wyWX/22WeLbOeiw1AfEBThB4Ii/EBQhB8IivADQRF+IKgiftWHnB577LFkfcuWLcn6ypUrW9aWL1/eTUs9sWbNmmR9165dyfrdd9+dqx4dZ34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx/j7w4osv5tr+9OnTBXVSvMOHD7es7d69O9e+243jZ9eaQAuc+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMb5e2Dnzp3J+rvvvptr/4sXL861fZlS1yr46KOPktuuXbs2Wb/lllu66gkTOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBtx/nNbFjSNyQdd/cbsmXzJP1S0oCkUUl3uvuH5bU5vZ05cyZZd/cedVK8I0eOJOtbt27tet833nhjsj5r1qyu943Ozvw/k3TbBcselPSau18t6bXsMYBppG343f1NSScvWDwoaSS7PyLp9oL7AlCybt/zX+HuY5KU3S4oriUAvVD6B35mttHMGmbWaDabZR8OQIe6Df+4mS2SpOz2eKsV3X3I3evuXq/Val0eDkDRug3/TkkbsvsbJO0oph0AvdI2/Gb2nKT/knSNmR01s+9IelTSrWZ2QNKt2WMA00jbcX53X9eixI+pM2fPnk3WR0ZGkvV2li1blqxffvnlufafx0svvZSsnzt3rmWt3Tj9+vXru+oJneEbfkBQhB8IivADQRF+ICjCDwRF+IGguHR3AQ4dOpSsv/zyy7n2v3r16mR97ty5ufaf8v777yfrjzzySNf73rRpU7Lez5ckvxhw5geCIvxAUIQfCIrwA0ERfiAowg8ERfiBoBjnL8C2bdtK3f/g4GCp+085efLCa7d+3ocfdn/F9ir/u8CZHwiL8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpy/AHnGujuxY0d6TpRFixaVduzt27eXtu/33nsvWb/kkktKO3a7y51fc801pR27X3DmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGg2o7zm9mwpG9IOu7uN2TLtkj6rqRmttpmd3+lrCb73bXXXlvq/oeHh3PV+9X9999f2bHbjfPv27cvWV+6dGmR7VSikzP/zyTdNsXyn7j7iuxf2OAD01Xb8Lv7m5LSl3MBMO3kec+/ycx+b2bDZlbefFEAStFt+LdK+qqkFZLGJP2o1YpmttHMGmbWaDabrVYD0GNdhd/dx939rLufk/RTSasS6w65e93d67Vards+ARSsq/Cb2eSfkX1TUvqjUQB9p5OhvuckrZY038yOSvqhpNVmtkKSSxqVVN2YDYCutA2/u6+bYvHTJfQybd13333J+s0335ysr1s31VP8/z744INk/dSpUy1rZV9roJ2FCxe2rM2dW+7nxKnrAdx7773JbefMmVNwN/2Hb/gBQRF+ICjCDwRF+IGgCD8QFOEHguLS3QUws2S93WWg9+zZk6y3myY7NRQ4NjaW3Paee+5J1g8fPpysL1u2LFl/4403WtaWLFmS3Bbl4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzj8NzJs3r+v6jBkzktt+/PHHXfV03uDgYLLOWH7/4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzn+Re+utt5L1EydO5Nr/XXfdlWt7VIczPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E1Xac38yWSnpG0kJJ5yQNufvjZjZP0i8lDUgalXSnu1c7HzS+4NVXX821/YoVK5L15cuX59o/qtPJmf8zST9w92sl/Z2k75nZdZIelPSau18t6bXsMYBpom343X3M3fdk909J2i9psaRBSSPZaiOSbi+rSQDF+1Lv+c1sQNJKSb+TdIW7j0kTfyAkLSi6OQDl6Tj8ZjZb0nZJ33f3P3+J7TaaWcPMGs1ms5seAZSgo/Cb2UxNBP/n7v7rbPG4mS3K6oskHZ9qW3cfcve6u9drtVoRPQMoQNvw28QUtE9L2u/uP55U2ilpQ3Z/g6QdxbcHoCyd/KT3JknrJe01s3eyZZslPSrpV2b2HUl/kvStclpEO59++mnL2vj4eK59z5w5M1mfNWtWrv2jOm3D7+6/ldRqAvpbim0HQK/wDT8gKMIPBEX4gaAIPxAU4QeCIvxAUFy6+yKQmmb79ddfz7XvtWvXJuuNRiNZT00RvnLlyq56QjE48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzI+mhhx7KVd+7d2+R7aBAnPmBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjG+S8CqWvnX3XVVcltDx48mKwvWJCegvHhhx9O1q+//vpkHdXhzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQbUd5zezpZKekbRQ0jlJQ+7+uJltkfRdSc1s1c3u/kpZjaK1Sy+9tGXtwIEDPewE00knX/L5TNIP3H2Pmc2R9LaZ7cpqP3H3fy2vPQBlaRt+dx+TNJbdP2Vm+yUtLrsxAOX6Uu/5zWxA0kpJv8sWbTKz35vZsJnNbbHNRjNrmFmj2WxOtQqACnQcfjObLWm7pO+7+58lbZX0VUkrNPHK4EdTbefuQ+5ed/d6rVYroGUARego/GY2UxPB/7m7/1qS3H3c3c+6+zlJP5W0qrw2ARStbfjNzCQ9LWm/u/940vJFk1b7pqR9xbcHoCydfNp/k6T1kvaa2TvZss2S1pnZCkkuaVTS/aV0CKAUnXza/1tJNkWJMX1gGuMbfkBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaDM3Xt3MLOmpMOTFs2XdKJnDXw5/dpbv/Yl0Vu3iuztb9y9o+vl9TT8Xzi4WcPd65U1kNCvvfVrXxK9dauq3njZDwRF+IGgqg7/UMXHT+nX3vq1L4neulVJb5W+5wdQnarP/AAqUkn4zew2M/sfMztoZg9W0UMrZjZqZnvN7B0za1Tcy7CZHTezfZOWzTOzXWZ2ILudcpq0inrbYmb/mz1375jZP1bU21Ize93M9pvZH8zsgWx5pc9doq9Knreev+w3sxmS/ijpVklHJe2WtM7d/7unjbRgZqOS6u5e+Ziwmd0s6bSkZ9z9hmzZv0g66e6PZn8457r7P/VJb1skna565uZsQplFk2eWlnS7pHtV4XOX6OtOVfC8VXHmXyXpoLsfcvczkrZJGqygj77n7m9KOnnB4kFJI9n9EU38z9NzLXrrC+4+5u57svunJJ2fWbrS5y7RVyWqCP9iSUcmPT6q/pry2yX9xszeNrONVTczhSuyadPPT5++oOJ+LtR25uZeumBm6b557rqZ8bpoVYR/qtl/+mnI4SZ3/1tJX5f0vezlLTrT0czNvTLFzNJ9odsZr4tWRfiPSlo66fESSccq6GNK7n4suz0u6QX13+zD4+cnSc1uj1fcz1/008zNU80srT547vppxusqwr9b0tVm9hUzmyXp25J2VtDHF5jZZdkHMTKzyyStUf/NPrxT0obs/gZJOyrs5XP6ZebmVjNLq+Lnrt9mvK7kSz7ZUMa/SZohadjd/7nnTUzBzJZp4mwvTUxi+osqezOz5ySt1sSvvsYl/VDSi5J+JelKSX+S9C137/kHby16W62Jl65/mbn5/HvsHvf295LekrRX0rls8WZNvL+u7LlL9LVOFTxvfMMPCIpv+AFBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCOr/AADq1C4+dPlaAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Route:{}\n", + "{\n", + " \"0\": \"0.08\",\n", + " \"1\": \"0.10\",\n", + " \"2\": \"0.09\",\n", + " \"3\": \"0.09\",\n", + " \"4\": \"0.12\",\n", + " \"5\": \"0.09\",\n", + " \"6\": \"0.10\",\n", + " \"7\": \"0.11\",\n", + " \"8\": \"0.09\",\n", + " \"9\": \"0.12\"\n", + "}\n" + ] + } + ], + "source": [ + "utils.predict_rest_mnist(mnist,\"mnist-classifier\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "node \"minikube\" labeled\r\n" + ] + } + ], + "source": [ + "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME: loadtest\n", + "LAST DEPLOYED: Tue Jun 19 18:53:23 2018\n", + "NAMESPACE: seldon\n", + "STATUS: DEPLOYED\n", + "\n", + "RESOURCES:\n", + "==> v1/Pod(related)\n", + "NAME READY STATUS RESTARTS AGE\n", + "locust-slave-1-jmjbs 0/2 Init:0/1 0 0s\n", + "locust-master-1-6xhvd 0/2 Init:0/1 0 0s\n", + "\n", + "==> v1/ReplicationController\n", + "NAME DESIRED CURRENT READY AGE\n", + "locust-slave-1 1 1 0 0s\n", + "locust-master-1 1 1 0 0s\n", + "\n", + "==> v1/Service\n", + "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", + "locust-master-1 NodePort 10.102.138.69 5557:31388/TCP,5558:32352/TCP,8089:30904/TCP 0s\n", + "\n", + "\n" + ] + } + ], + "source": [ + "!helm install seldon-core-loadtesting --name loadtest \\\n", + " --namespace seldon \\\n", + " --repo https://storage.googleapis.com/seldon-charts \\\n", + " --set locust.script=mnist_rest_locust.py \\\n", + " --set locust.host=http://mnist-deployment:8000 \\\n", + " --set oauth.enabled=false \\\n", + " --set oauth.key=oauth-key \\\n", + " --set oauth.secret=oauth-secret \\\n", + " --set locust.hatchRate=1 \\\n", + " --set locust.clients=1 \\\n", + " --set loadtest.sendFeedback=1 \\\n", + " --set locust.minWait=0 \\\n", + " --set locust.maxWait=0 \\\n", + " --set replicaCount=1 \\\n", + " --set data.size=784\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize istio_canary_v1.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!istioctl create -f istio_canary_v1.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!kubectl apply -f mnist_v2.json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for i in range (100):\n", + " utils.predict_rest_mnist(mnist,\"mnist-classifier\")" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Updated config virtual-service/seldon/mnist-deployment to revision 3005\r\n", + "Updated config destination-rule/seldon/mnist-deployment to revision 3006\r\n" + ] + } + ], + "source": [ + "!istioctl replace -f istio_canary_v2.yaml" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tear Down" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "release \"loadtest\" deleted\r\n" + ] + } + ], + "source": [ + "!helm delete loadtest --purge" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/istio/canary_update/istio_canary_v1.yaml b/examples/istio/canary_update/istio_canary_v1.yaml new file mode 100644 index 0000000000..80902e7e35 --- /dev/null +++ b/examples/istio/canary_update/istio_canary_v1.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: mnist-deployment + namespace: seldon +spec: + hosts: + - mnist-deployment + http: + - route: + - destination: + host: mnist-deployment + subset: v1 +--- +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: mnist-deployment + namespace: seldon +spec: + host: mnist-deployment + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 +--- + diff --git a/examples/istio/canary_update/istio_canary_v2.yaml b/examples/istio/canary_update/istio_canary_v2.yaml new file mode 100644 index 0000000000..0be2b8818f --- /dev/null +++ b/examples/istio/canary_update/istio_canary_v2.yaml @@ -0,0 +1,35 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: mnist-deployment + namespace: seldon +spec: + hosts: + - mnist-deployment + http: + - route: + - destination: + host: mnist-deployment + subset: v1 + weight: 90 + - destination: + host: mnist-deployment + subset: v2 + weight: 10 +--- +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: mnist-deployment + namespace: seldon +spec: + host: mnist-deployment + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 +--- + diff --git a/examples/istio/canary_update/istio_canary_v3.yaml b/examples/istio/canary_update/istio_canary_v3.yaml new file mode 100644 index 0000000000..ad1e73afa4 --- /dev/null +++ b/examples/istio/canary_update/istio_canary_v3.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: mnist-deployment + namespace: seldon +spec: + hosts: + - mnist-deployment + http: + - route: + - destination: + host: mnist-deployment + subset: v2 +--- +apiVersion: networking.istio.io/v1alpha3 +kind: DestinationRule +metadata: + name: mnist-deployment + namespace: seldon +spec: + host: mnist-deployment + subsets: + - name: v1 + labels: + version: v1 + - name: v2 + labels: + version: v2 +--- + diff --git a/examples/istio/canary_update/mnist_v1.json b/examples/istio/canary_update/mnist_v1.json new file mode 100644 index 0000000000..d56e7a4d50 --- /dev/null +++ b/examples/istio/canary_update/mnist_v1.json @@ -0,0 +1,52 @@ +{ + "apiVersion": "machinelearning.seldon.io/v1alpha2", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "mnist-classifier" + }, + "spec": { + "annotations": { + "project_name": "Mnist classification" + }, + "name": "mnist-deployment", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpecs": [{ + "spec": { + "containers": [ + { + "image": "seldonio/r-mnist:0.1", + "imagePullPolicy": "IfNotPresent", + "name": "r-mnist-classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "graph": { + "children": [], + "name": "r-mnist-classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "r-mnist-predictor", + "replicas": 1, + "labels":{ + "version":"v1" + } + } + ] + } +} diff --git a/examples/istio/canary_update/mnist_v2.json b/examples/istio/canary_update/mnist_v2.json new file mode 100644 index 0000000000..7599e46d7f --- /dev/null +++ b/examples/istio/canary_update/mnist_v2.json @@ -0,0 +1,85 @@ +{ + "apiVersion": "machinelearning.seldon.io/v1alpha2", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "mnist-classifier" + }, + "spec": { + "annotations": { + "project_name": "Mnist classification" + }, + "name": "mnist-deployment", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpecs": [{ + "spec": { + "containers": [ + { + "image": "seldonio/r-mnist:0.1", + "imagePullPolicy": "IfNotPresent", + "name": "r-mnist-classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "graph": { + "children": [], + "name": "r-mnist-classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "r-mnist-predictor", + "replicas": 1, + "labels":{ + "version":"v1" + } + }, + { + "componentSpecs": [{ + "spec": { + "containers": [ + { + "image": "seldonio/deep-mnist:0.1", + "imagePullPolicy": "IfNotPresent", + "name": "tf-mnist-classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "graph": { + "children": [], + "name": "tf-mnist-classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "tf-mnist-predictor", + "replicas": 1, + "labels":{ + "version":"v2" + } + } + + ] + } +} diff --git a/examples/istio/canary_update/utils.py b/examples/istio/canary_update/utils.py new file mode 100644 index 0000000000..df5bb6018a --- /dev/null +++ b/examples/istio/canary_update/utils.py @@ -0,0 +1,83 @@ +import requests +from requests.auth import HTTPBasicAuth +from random import randint,random +import json +from matplotlib import pyplot as plt +import numpy as np +from tensorflow.examples.tutorials.mnist import input_data + + +AMBASSADOR_API_IP="localhost:8002" +API_HTTP="localhost:8003" + +def get_token(): + payload = {'grant_type': 'client_credentials'} + response = requests.post( + "http://"+API_HTTP+"/oauth/token", + auth=HTTPBasicAuth('oauth-key', 'oauth-secret'), + data=payload) + print(response.text) + token = response.json()["access_token"] + return token + +def rest_request_seldon(request): + token = get_token() + headers = {'Authorization': 'Bearer '+token} + response = requests.post( + "http://"+API_HTTP+"/api/v0.1/predictions", + headers=headers, + json=request) + return response.json() + + +def rest_request(deploymentName,request): + response = requests.post( + "http://"+AMBASSADOR_API_IP+"/seldon/"+deploymentName+"/api/v0.1/predictions", + json=request) + return response.json() + +def rest_request_auth(deploymentName,data,username,password): + payload = {"data":{"ndarray":data.tolist()}} + response = requests.post( + "http://"+AMBASSADOR_API_IP+"/seldon/"+deploymentName+"/api/v0.1/predictions", + json=payload, + auth=HTTPBasicAuth(username, password)) + print(response.status_code) + return response.json() + +def send_feedback_rest(deploymentName,request,response,reward): + feedback = { + "request": request, + "response": response, + "reward": reward + } + ret = requests.post( + "http://"+AMBASSADOR_API_IP+"/seldon/"+deploymentName+"/api/v0.1/feedback", + json=feedback) + return ret.text + +def gen_image(arr): + two_d = (np.reshape(arr, (28, 28)) * 255).astype(np.uint8) + plt.imshow(two_d,cmap=plt.cm.gray_r, interpolation='nearest') + return plt + +def download_mnist(): + return input_data.read_data_sets("MNIST_data/", one_hot = True) + +def predict_rest_mnist(mnist,deployment_name): + batch_xs, batch_ys = mnist.train.next_batch(1) + chosen=0 + gen_image(batch_xs[chosen]).show() + data = batch_xs[chosen].reshape((1,784)) + features = ["X"+str(i+1) for i in range (0,784)] + request = {"data":{"names":features,"ndarray":data.tolist()}} + predictions = rest_request(deployment_name,request) + #predictions = rest_request_seldon(request) + print("Route:"+json.dumps(predictions["meta"]["routing"],indent=2)) + fpreds = [ '%.2f' % elem for elem in predictions["data"]["ndarray"][0] ] + m = dict(zip(predictions["data"]["names"],fpreds)) + print(json.dumps(m,indent=2)) + + + + From 0fed1ea04706d144297cf03da39d3eb1530d7cad Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Wed, 20 Jun 2018 16:34:37 +0100 Subject: [PATCH 15/20] updated istio example for canary --- api-frontend/pom.xml | 2 +- cluster-manager/pom.xml | 2 +- engine/pom.xml | 2 +- examples/istio/canary_update/README.md | 5 + examples/istio/canary_update/canary.ipynb | 695 +++++++----------- .../istio/canary_update/istio_canary_v2.yaml | 4 +- helm-charts/seldon-core-crd/Chart.yaml | 2 +- helm-charts/seldon-core/Chart.yaml | 2 +- helm-charts/seldon-core/values.yaml | 6 +- .../seldon-core/prototypes/core.jsonnet | 6 +- 10 files changed, 288 insertions(+), 438 deletions(-) create mode 100644 examples/istio/canary_update/README.md diff --git a/api-frontend/pom.xml b/api-frontend/pom.xml index 8da42d37ff..14d6d6124c 100644 --- a/api-frontend/pom.xml +++ b/api-frontend/pom.xml @@ -10,7 +10,7 @@ io.seldon.apife seldon-apife - 0.1.8-SNAPSHOT + 0.1.8-ISTIO-SNAPSHOT jar api-frontend diff --git a/cluster-manager/pom.xml b/cluster-manager/pom.xml index 110918b5ce..d5b028d2c4 100644 --- a/cluster-manager/pom.xml +++ b/cluster-manager/pom.xml @@ -4,7 +4,7 @@ io.seldon.clustermanager seldon-cluster-manager jar - 0.1.8-SNAPSHOT + 0.1.8-ISTIO-SNAPSHOT seldon-cluster-manager http://maven.apache.org diff --git a/engine/pom.xml b/engine/pom.xml index 644c03b18e..f86681b404 100644 --- a/engine/pom.xml +++ b/engine/pom.xml @@ -10,7 +10,7 @@ io.seldon.engine seldon-engine - 0.1.8-SNAPSHOT + 0.1.8-ISTIO-SNAPSHOT jar engine diff --git a/examples/istio/canary_update/README.md b/examples/istio/canary_update/README.md new file mode 100644 index 0000000000..f593fbd4b3 --- /dev/null +++ b/examples/istio/canary_update/README.md @@ -0,0 +1,5 @@ +# Canary Roll Out using Seldon-Core and Istio + +This folder provides resources to illustrate how to do a canary roll out of one MNIST model to another using the canary pattern where a small amount of traffic is sent to the new model to validate it before sending all traffic to the new model. + +There is a [Jupyter Notebook](canary.ipynb) that provides a step by step demo. \ No newline at end of file diff --git a/examples/istio/canary_update/canary.ipynb b/examples/istio/canary_update/canary.ipynb index 45e4cb0ae2..988a2ccc87 100644 --- a/examples/istio/canary_update/canary.ipynb +++ b/examples/istio/canary_update/canary.ipynb @@ -1,242 +1,158 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Canary Roll Out of ML Models with Seldon and Istio\n", + "\n", + "This folder provides resources to illustrate how to do a canary\troll out of one\tMNIST model to another using the canary\tpattern where a small amount\tof traffic is sent to the new model to validate\tit before sending all traffic to the new model.\n", + "\n", + "We utilize two MNIST digit classification models. \n", + "\n", + " * Version 1 of the model using R\n", + " * Version 2 of the model using Tensorflow.\n", + " \n", + "After deploying Istio and Seldon to a kubernetes cluster we will:\n", + "\n", + " * Deploy version 1 R based model using seldon-core\n", + " * Create an istio routing rule to direct all traffic to this version\n", + " * Create a canary deployment with both version 1 and version 2 (the Tensorflow model)\n", + " * Update the istio routing to send 10% of the traffic to version 2\n", + " * Update the istio routing to send 100% of the traffic to version 2\n", + " \n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup\n", + "\n", + "The steps below will install istio and seldon onto a GKE Cluster. If you wish to use your own setup then you need to ensure\n", + "\n", + " * You allow istio egress to the internet as the load test downloads MNIST images\n", + " * Ensure you give your user cluster-admin privledges\n", + " * Install seldon into a namespace seldon\n", + " \n", + "To follow the steps below you will need:\n", + " \n", + " * a Google project running a K8S cluster\n", + " * gcloud and kubectl installed\n", + " * istio download" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Environment Variables" + ] + }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "env: ISTIO_HOME=/home/clive/work/istio/istio-0.8.0\n" - ] - } - ], + "outputs": [], "source": [ "%env ISTIO_HOME=/home/clive/work/istio/istio-0.8.0" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%env ZONE=europe-west1-d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%env PROJECT=my-project" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Determine CIDR ranges" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gcloud container clusters describe cluster-istio-1 --zone ${ZONE} --project ${PROJECT} | grep -e clusterIpv4Cidr -e servicesIpv4Cidr" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Helm" + ] + }, + { + "cell_type": "code", + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "serviceaccount \"tiller\" created\r\n", - "clusterrolebinding \"tiller\" created\r\n" - ] - } - ], + "outputs": [], + "source": [ + "!kubectl create clusterrolebinding my-cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud info --format=\"value(config.account)\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!kubectl create -f ${ISTIO_HOME}/install/kubernetes/helm/helm-service-account.yaml" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "$HELM_HOME has been configured at /home/clive/.helm.\n", - "\n", - "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n", - "Happy Helming!\n" - ] - } - ], + "outputs": [], "source": [ "!helm init --service-account tiller" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Istio\n", + "\n", + "** Replace the CIDR values with those you got above **\n", + "\n", + "For more details see [istio docs on egress](https://istio.io/docs/tasks/traffic-management/egress/#calling-external-services-directly)" + ] + }, { "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME: istio\n", - "LAST DEPLOYED: Tue Jun 19 18:45:17 2018\n", - "NAMESPACE: istio-system\n", - "STATUS: DEPLOYED\n", - "\n", - "RESOURCES:\n", - "==> v1beta1/CustomResourceDefinition\n", - "NAME AGE\n", - "deniers.config.istio.io 45s\n", - "rbacs.config.istio.io 45s\n", - "prometheuses.config.istio.io 45s\n", - "servicecontrolreports.config.istio.io 45s\n", - "logentries.config.istio.io 45s\n", - "listentries.config.istio.io 45s\n", - "kuberneteses.config.istio.io 45s\n", - "fluentds.config.istio.io 45s\n", - "kubernetesenvs.config.istio.io 45s\n", - "stdios.config.istio.io 45s\n", - "circonuses.config.istio.io 45s\n", - "listcheckers.config.istio.io 45s\n", - "solarwindses.config.istio.io 45s\n", - "apikeys.config.istio.io 45s\n", - "rules.config.istio.io 45s\n", - "quotas.config.istio.io 45s\n", - "reportnothings.config.istio.io 45s\n", - "tracespans.config.istio.io 45s\n", - "serviceroles.config.istio.io 45s\n", - "servicerolebindings.config.istio.io 45s\n", - "noops.config.istio.io 45s\n", - "opas.config.istio.io 45s\n", - "statsds.config.istio.io 45s\n", - "checknothings.config.istio.io 45s\n", - "attributemanifests.config.istio.io 45s\n", - "memquotas.config.istio.io 45s\n", - "stackdrivers.config.istio.io 45s\n", - "authorizations.config.istio.io 45s\n", - "metrics.config.istio.io 45s\n", - "servicecontrols.config.istio.io 45s\n", - "routerules.config.istio.io 45s\n", - "quotaspecbindings.config.istio.io 45s\n", - "destinationpolicies.config.istio.io 45s\n", - "egressrules.config.istio.io 45s\n", - "httpapispecs.config.istio.io 45s\n", - "quotaspecs.config.istio.io 45s\n", - "httpapispecbindings.config.istio.io 45s\n", - "gateways.networking.istio.io 45s\n", - "destinationrules.networking.istio.io 45s\n", - "virtualservices.networking.istio.io 45s\n", - "serviceentries.networking.istio.io 45s\n", - "policies.authentication.istio.io 45s\n", - "\n", - "==> v1beta1/RoleBinding\n", - "NAME AGE\n", - "istio-cleanup-old-ca-istio-system 45s\n", - "\n", - "==> v2beta1/HorizontalPodAutoscaler\n", - "NAME REFERENCE TARGETS MINPODS MAXPODS REPLICAS AGE\n", - "istio-egressgateway Deployment/istio-egressgateway / 80% 1 1 1 45s\n", - "istio-ingress Deployment/istio-ingress / 80% 1 1 1 45s\n", - "istio-ingressgateway Deployment/istio-ingressgateway / 80% 1 1 1 45s\n", - "\n", - "==> v1beta1/MutatingWebhookConfiguration\n", - "NAME AGE\n", - "istio-sidecar-injector 45s\n", - "\n", - "==> v1/Pod(related)\n", - "NAME READY STATUS RESTARTS AGE\n", - "istio-egressgateway-58d98d898c-ftrxw 1/1 Running 0 45s\n", - "istio-ingress-6fb78f687f-m4c47 1/1 Running 0 45s\n", - "istio-ingressgateway-6bc7c7c4bc-km9xn 0/1 ContainerCreating 0 45s\n", - "istio-telemetry-54b5bf4847-ndp9d 2/2 Running 0 45s\n", - "istio-policy-5c7fbb4b9f-mdwcl 0/2 ContainerCreating 0 45s\n", - "istio-statsd-prom-bridge-6dbb7dcc7f-qxhxg 0/1 ContainerCreating 0 45s\n", - "istio-pilot-6c5c6b586c-xgp4q 0/2 ContainerCreating 0 45s\n", - "prometheus-586d95b8d9-6wfbj 0/1 ContainerCreating 0 45s\n", - "istio-citadel-ff5696f6f-78mt5 0/1 ContainerCreating 0 44s\n", - "istio-sidecar-injector-dbd67c88d-glfbf 0/1 ContainerCreating 0 44s\n", - "\n", - "==> v1/ConfigMap\n", - "NAME DATA AGE\n", - "istio-statsd-prom-bridge 1 45s\n", - "istio-mixer-custom-resources 1 45s\n", - "prometheus 1 45s\n", - "istio 1 45s\n", - "istio-sidecar-injector 1 45s\n", - "\n", - "==> v1/ServiceAccount\n", - "NAME SECRETS AGE\n", - "istio-egressgateway-service-account 1 45s\n", - "istio-ingress-service-account 1 45s\n", - "istio-ingressgateway-service-account 1 45s\n", - "istio-mixer-post-install-account 1 45s\n", - "istio-mixer-service-account 1 45s\n", - "istio-pilot-service-account 1 45s\n", - "prometheus 1 45s\n", - "istio-cleanup-old-ca-service-account 1 45s\n", - "istio-citadel-service-account 1 45s\n", - "istio-sidecar-injector-service-account 1 45s\n", - "\n", - "==> v1beta1/ClusterRole\n", - "NAME AGE\n", - "istio-ingress-istio-system 45s\n", - "istio-mixer-istio-system 45s\n", - "istio-mixer-post-install-istio-system 45s\n", - "istio-pilot-istio-system 45s\n", - "prometheus-istio-system 45s\n", - "istio-citadel-istio-system 45s\n", - "istio-sidecar-injector-istio-system 45s\n", - "\n", - "==> v1beta1/ClusterRoleBinding\n", - "NAME AGE\n", - "istio-ingress-istio-system 45s\n", - "istio-mixer-admin-role-binding-istio-system 45s\n", - "istio-mixer-post-install-role-binding-istio-system 45s\n", - "istio-pilot-istio-system 45s\n", - "prometheus-istio-system 45s\n", - "istio-citadel-istio-system 45s\n", - "istio-sidecar-injector-admin-role-binding-istio-system 45s\n", - "\n", - "==> v1beta1/Role\n", - "NAME AGE\n", - "istio-cleanup-old-ca-istio-system 45s\n", - "\n", - "==> v1/Service\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "istio-egressgateway ClusterIP 10.102.213.53 80/TCP,443/TCP 45s\n", - "istio-ingress LoadBalancer 10.106.70.233 80:32000/TCP,443:31186/TCP 45s\n", - "istio-ingressgateway LoadBalancer 10.106.229.108 80:31380/TCP,443:31390/TCP,31400:31400/TCP 45s\n", - "istio-policy ClusterIP 10.111.134.24 9091/TCP,15004/TCP,9093/TCP 45s\n", - "istio-telemetry ClusterIP 10.98.139.75 9091/TCP,15004/TCP,9093/TCP,42422/TCP 45s\n", - "istio-statsd-prom-bridge ClusterIP 10.101.127.9 9102/TCP,9125/UDP 45s\n", - "istio-pilot ClusterIP 10.108.21.76 15003/TCP,15005/TCP,15007/TCP,15010/TCP,15011/TCP,8080/TCP,9093/TCP 45s\n", - "prometheus ClusterIP 10.109.91.39 9090/TCP 45s\n", - "istio-citadel ClusterIP 10.98.205.164 8060/TCP,9093/TCP 45s\n", - "istio-sidecar-injector ClusterIP 10.103.0.45 443/TCP 45s\n", - "\n", - "==> v1beta1/Deployment\n", - "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", - "istio-egressgateway 1 1 1 1 45s\n", - "istio-ingress 1 1 1 1 45s\n", - "istio-ingressgateway 1 1 1 0 45s\n", - "istio-telemetry 1 1 1 1 45s\n", - "istio-policy 1 1 1 0 45s\n", - "istio-statsd-prom-bridge 1 1 1 0 45s\n", - "istio-pilot 1 1 1 0 45s\n", - "prometheus 1 1 1 0 45s\n", - "istio-citadel 1 1 1 0 45s\n", - "istio-sidecar-injector 1 1 1 0 45s\n", - "\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!helm install ${ISTIO_HOME}/install/kubernetes/helm/istio --name istio --namespace istio-system \\\n", - " --set global.proxy.includeIPRanges=\"10.0.0.1/8\"" + " --set global.proxy.includeIPRanges=\"10.20.0.0/14\\,10.23.240.0/20\"" ] }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "service \"grafana\" created\n", - "deployment \"grafana\" created\n", - "serviceaccount \"grafana\" created\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl apply -f ${ISTIO_HOME}/install/kubernetes/addons/grafana.yaml" ] @@ -245,12 +161,20 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "To view the istio Grafana dashboard:\n", "```\n", "kubectl -n istio-system port-forward $(kubectl -n istio-system get pod -l app=grafana -o jsonpath='{.items[0].metadata.name}') 3000:3000\n", "```\n", "http://localhost:3000/dashboard/db/istio-dashboard" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Install Seldon" + ] + }, { "cell_type": "code", "execution_count": null, @@ -266,106 +190,43 @@ "metadata": {}, "outputs": [], "source": [ - "!kubectl label namespace seldon istio-injection=enabled" + "!kubectl apply -f ../../../notebooks/resources/ambassador-rbac.yaml -n seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To send requests to Ambassador ingress:\n", + " \n", + "```\n", + "kubectl port-forward $(kubectl get pods -n seldon -l service=ambaador -o jsonpath='{.items[0].metadata.name}') -n seldon 8002:80\n", + "```" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "clusterrole \"ambassador\" created\n", - "serviceaccount \"ambassador\" created\n", - "clusterrolebinding \"ambassador\" created\n", - "service \"ambassador\" created\n", - "service \"ambassador-admin\" created\n", - "deployment \"ambassador\" created\n" - ] - } - ], + "outputs": [], "source": [ - "!kubectl apply -f ../../../notebooks/resources/ambassador-rbac.yaml -n seldon" + "!helm install ../../../helm-charts/seldon-core-crd --name seldon-core-crd" ] }, { "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME: seldon-core-analytics\n", - "LAST DEPLOYED: Tue Jun 19 18:49:33 2018\n", - "NAMESPACE: seldon\n", - "STATUS: DEPLOYED\n", - "\n", - "RESOURCES:\n", - "==> v1/Service\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "alertmanager ClusterIP 10.97.10.96 80/TCP 1s\n", - "grafana-prom NodePort 10.101.14.81 80:31040/TCP 1s\n", - "prometheus-node-exporter ClusterIP None 9100/TCP 1s\n", - "prometheus-seldon ClusterIP 10.103.146.190 80/TCP 1s\n", - "\n", - "==> v1/Pod(related)\n", - "NAME READY STATUS RESTARTS AGE\n", - "grafana-prom-import-dashboards-mp82k 0/2 Init:0/1 0 1s\n", - "alertmanager-deployment-7fbfdfdfb6-tr8fj 0/2 Init:0/1 0 1s\n", - "grafana-prom-deployment-7b45fb85d4-htvvh 0/2 Init:0/1 0 1s\n", - "prometheus-node-exporter-wt5hk 0/1 Pending 0 1s\n", - "prometheus-deployment-cbfd78cc7-qwxx9 0/2 Pending 0 1s\n", - "\n", - "==> v1/ServiceAccount\n", - "NAME SECRETS AGE\n", - "prometheus 1 1s\n", - "\n", - "==> v1beta1/ClusterRole\n", - "NAME AGE\n", - "prometheus 1s\n", - "\n", - "==> v1beta1/ClusterRoleBinding\n", - "NAME AGE\n", - "prometheus 1s\n", - "\n", - "==> v1/Job\n", - "NAME DESIRED SUCCESSFUL AGE\n", - "grafana-prom-import-dashboards 1 0 1s\n", - "\n", - "==> v1beta1/Deployment\n", - "NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE\n", - "alertmanager-deployment 1 1 1 0 1s\n", - "grafana-prom-deployment 1 1 1 0 1s\n", - "prometheus-deployment 1 1 1 0 1s\n", - "\n", - "==> v1/Secret\n", - "NAME TYPE DATA AGE\n", - "grafana-prom-secret Opaque 1 1s\n", - "\n", - "==> v1/ConfigMap\n", - "NAME DATA AGE\n", - "alertmanager-server-conf 1 1s\n", - "grafana-import-dashboards 5 1s\n", - "prometheus-rules 4 1s\n", - "prometheus-server-conf 1 1s\n", - "\n", - "==> v1beta1/DaemonSet\n", - "NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE\n", - "prometheus-node-exporter 1 1 0 1 0 1s\n", - "\n", - "\n", - "NOTES:\n", - "NOTES: TODO\n", - "\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!helm install ../../../helm-charts/seldon-core --name seldon-core --namespace seldon" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!helm install ../../../helm-charts/seldon-core-analytics --name seldon-core-analytics \\\n", " --set grafana_prom_admin_password=password \\\n", @@ -377,6 +238,8 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "To view the Seldon Grafana dashboard:\n", + "\n", "```\n", "kubectl port-forward $(kubectl get pods -n seldon -l app=grafana-prom-server -o jsonpath='{.items[0].metadata.name}') -n seldon 3001:3000\n", "```\n", @@ -386,54 +249,49 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Context \"minikube\" modified.\r\n" - ] - } - ], + "outputs": [], "source": [ - "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + "!kubectl label namespace seldon istio-injection=enabled" ] }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "!mkdir -p proto\n", - "!cp ../../../proto/prediction.proto ./proto\n", - "!python -m grpc.tools.protoc -I. --python_out=. --grpc_python_out=. ./proto/prediction.proto" + "!kubectl config set-context $(kubectl config current-context) --namespace=seldon" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch Version 1 Model" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Extracting MNIST_data/train-images-idx3-ubyte.gz\n", - "Extracting MNIST_data/train-labels-idx1-ubyte.gz\n", - "Extracting MNIST_data/t10k-images-idx3-ubyte.gz\n", - "Extracting MNIST_data/t10k-labels-idx1-ubyte.gz\n" - ] - } - ], + "outputs": [], "source": [ "%matplotlib inline\n", "import utils\n", "mnist = utils.download_mnist()" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize mnist_v1.json" + ] + }, { "cell_type": "code", "execution_count": null, @@ -443,6 +301,13 @@ "!kubectl create -f mnist_v1.json" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "** Wait until new pods are running **" + ] + }, { "cell_type": "code", "execution_count": null, @@ -454,93 +319,34 @@ }, { "cell_type": "code", - "execution_count": 30, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAADbtJREFUeJzt3W+sVPWdx/HPd1lIVDCBMAgB3FuqMf4Jhc1INnFj2BiJ3TS5FlNTNIhJLT4oiU36YBVNypNNzLpt1wcrerve9GpaaQ1V0Bi3xBi1yaZhJFro4hYkl8JycxlEU9AHKHz3wT10r3jnN+Occ+bM5ft+JWRmzvf8+Trxc8/M/GbOz9xdAOL5q6obAFANwg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+IKi/7uXB5s+f7wMDA708JBDK6OioTpw4YZ2smyv8ZnabpMclzZD0H+7+aGr9gYEBNRqNPIcEkFCv1ztet+uX/WY2Q9K/S/q6pOskrTOz67rdH4DeyvOef5Wkg+5+yN3PSNomabCYtgCULU/4F0s6Munx0WzZ55jZRjNrmFmj2WzmOByAIuUJ/1QfKnzh98HuPuTudXev12q1HIcDUKQ84T8qaemkx0skHcvXDoBeyRP+3ZKuNrOvmNksSd+WtLOYtgCUreuhPnf/zMw2SfpPTQz1Dbv7HwrrDECpco3zu/srkl4pqBcAPcTXe4GgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwgq1yy9ZjYq6ZSks5I+c/d6EU0BknTmzJlk/YknnkjWn3rqqZa1/fv3d9XTxSRX+DP/4O4nCtgPgB7iZT8QVN7wu6TfmNnbZraxiIYA9Ebel/03ufsxM1sgaZeZvefub05eIfujsFGSrrzyypyHA1CUXGd+dz+W3R6X9IKkVVOsM+TudXev12q1PIcDUKCuw29ml5nZnPP3Ja2RtK+oxgCUK8/L/iskvWBm5/fzC3d/tZCuAJSu6/C7+yFJXyuwl7A++eSTZP35559P1u+4446WtdmzZ3fVUz944IEHkvUnn3wyWX/22WeLbOeiw1AfEBThB4Ii/EBQhB8IivADQRF+IKgiftWHnB577LFkfcuWLcn6ypUrW9aWL1/eTUs9sWbNmmR9165dyfrdd9+dqx4dZ34gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCIpx/j7w4osv5tr+9OnTBXVSvMOHD7es7d69O9e+243jZ9eaQAuc+YGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMb5e2Dnzp3J+rvvvptr/4sXL861fZlS1yr46KOPktuuXbs2Wb/lllu66gkTOPMDQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFBtx/nNbFjSNyQdd/cbsmXzJP1S0oCkUUl3uvuH5bU5vZ05cyZZd/cedVK8I0eOJOtbt27tet833nhjsj5r1qyu943Ozvw/k3TbBcselPSau18t6bXsMYBppG343f1NSScvWDwoaSS7PyLp9oL7AlCybt/zX+HuY5KU3S4oriUAvVD6B35mttHMGmbWaDabZR8OQIe6Df+4mS2SpOz2eKsV3X3I3evuXq/Val0eDkDRug3/TkkbsvsbJO0oph0AvdI2/Gb2nKT/knSNmR01s+9IelTSrWZ2QNKt2WMA00jbcX53X9eixI+pM2fPnk3WR0ZGkvV2li1blqxffvnlufafx0svvZSsnzt3rmWt3Tj9+vXru+oJneEbfkBQhB8IivADQRF+ICjCDwRF+IGguHR3AQ4dOpSsv/zyy7n2v3r16mR97ty5ufaf8v777yfrjzzySNf73rRpU7Lez5ckvxhw5geCIvxAUIQfCIrwA0ERfiAowg8ERfiBoBjnL8C2bdtK3f/g4GCp+085efLCa7d+3ocfdn/F9ir/u8CZHwiL8ANBEX4gKMIPBEX4gaAIPxAU4QeCYpy/AHnGujuxY0d6TpRFixaVduzt27eXtu/33nsvWb/kkktKO3a7y51fc801pR27X3DmB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGg2o7zm9mwpG9IOu7uN2TLtkj6rqRmttpmd3+lrCb73bXXXlvq/oeHh3PV+9X9999f2bHbjfPv27cvWV+6dGmR7VSikzP/zyTdNsXyn7j7iuxf2OAD01Xb8Lv7m5LSl3MBMO3kec+/ycx+b2bDZlbefFEAStFt+LdK+qqkFZLGJP2o1YpmttHMGmbWaDabrVYD0GNdhd/dx939rLufk/RTSasS6w65e93d67Vards+ARSsq/Cb2eSfkX1TUvqjUQB9p5OhvuckrZY038yOSvqhpNVmtkKSSxqVVN2YDYCutA2/u6+bYvHTJfQybd13333J+s0335ysr1s31VP8/z744INk/dSpUy1rZV9roJ2FCxe2rM2dW+7nxKnrAdx7773JbefMmVNwN/2Hb/gBQRF+ICjCDwRF+IGgCD8QFOEHguLS3QUws2S93WWg9+zZk6y3myY7NRQ4NjaW3Paee+5J1g8fPpysL1u2LFl/4403WtaWLFmS3Bbl4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzj8NzJs3r+v6jBkzktt+/PHHXfV03uDgYLLOWH7/4swPBEX4gaAIPxAU4QeCIvxAUIQfCIrwA0Exzn+Re+utt5L1EydO5Nr/XXfdlWt7VIczPxAU4QeCIvxAUIQfCIrwA0ERfiAowg8E1Xac38yWSnpG0kJJ5yQNufvjZjZP0i8lDUgalXSnu1c7HzS+4NVXX821/YoVK5L15cuX59o/qtPJmf8zST9w92sl/Z2k75nZdZIelPSau18t6bXsMYBpom343X3M3fdk909J2i9psaRBSSPZaiOSbi+rSQDF+1Lv+c1sQNJKSb+TdIW7j0kTfyAkLSi6OQDl6Tj8ZjZb0nZJ33f3P3+J7TaaWcPMGs1ms5seAZSgo/Cb2UxNBP/n7v7rbPG4mS3K6oskHZ9qW3cfcve6u9drtVoRPQMoQNvw28QUtE9L2u/uP55U2ilpQ3Z/g6QdxbcHoCyd/KT3JknrJe01s3eyZZslPSrpV2b2HUl/kvStclpEO59++mnL2vj4eK59z5w5M1mfNWtWrv2jOm3D7+6/ldRqAvpbim0HQK/wDT8gKMIPBEX4gaAIPxAU4QeCIvxAUFy6+yKQmmb79ddfz7XvtWvXJuuNRiNZT00RvnLlyq56QjE48wNBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIzzI+mhhx7KVd+7d2+R7aBAnPmBoAg/EBThB4Ii/EBQhB8IivADQRF+ICjG+S8CqWvnX3XVVcltDx48mKwvWJCegvHhhx9O1q+//vpkHdXhzA8ERfiBoAg/EBThB4Ii/EBQhB8IivADQbUd5zezpZKekbRQ0jlJQ+7+uJltkfRdSc1s1c3u/kpZjaK1Sy+9tGXtwIEDPewE00knX/L5TNIP3H2Pmc2R9LaZ7cpqP3H3fy2vPQBlaRt+dx+TNJbdP2Vm+yUtLrsxAOX6Uu/5zWxA0kpJv8sWbTKz35vZsJnNbbHNRjNrmFmj2WxOtQqACnQcfjObLWm7pO+7+58lbZX0VUkrNPHK4EdTbefuQ+5ed/d6rVYroGUARego/GY2UxPB/7m7/1qS3H3c3c+6+zlJP5W0qrw2ARStbfjNzCQ9LWm/u/940vJFk1b7pqR9xbcHoCydfNp/k6T1kvaa2TvZss2S1pnZCkkuaVTS/aV0CKAUnXza/1tJNkWJMX1gGuMbfkBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaDM3Xt3MLOmpMOTFs2XdKJnDXw5/dpbv/Yl0Vu3iuztb9y9o+vl9TT8Xzi4WcPd65U1kNCvvfVrXxK9dauq3njZDwRF+IGgqg7/UMXHT+nX3vq1L4neulVJb5W+5wdQnarP/AAqUkn4zew2M/sfMztoZg9W0UMrZjZqZnvN7B0za1Tcy7CZHTezfZOWzTOzXWZ2ILudcpq0inrbYmb/mz1375jZP1bU21Ize93M9pvZH8zsgWx5pc9doq9Knreev+w3sxmS/ijpVklHJe2WtM7d/7unjbRgZqOS6u5e+Ziwmd0s6bSkZ9z9hmzZv0g66e6PZn8457r7P/VJb1skna565uZsQplFk2eWlnS7pHtV4XOX6OtOVfC8VXHmXyXpoLsfcvczkrZJGqygj77n7m9KOnnB4kFJI9n9EU38z9NzLXrrC+4+5u57svunJJ2fWbrS5y7RVyWqCP9iSUcmPT6q/pry2yX9xszeNrONVTczhSuyadPPT5++oOJ+LtR25uZeumBm6b557rqZ8bpoVYR/qtl/+mnI4SZ3/1tJX5f0vezlLTrT0czNvTLFzNJ9odsZr4tWRfiPSlo66fESSccq6GNK7n4suz0u6QX13+zD4+cnSc1uj1fcz1/008zNU80srT547vppxusqwr9b0tVm9hUzmyXp25J2VtDHF5jZZdkHMTKzyyStUf/NPrxT0obs/gZJOyrs5XP6ZebmVjNLq+Lnrt9mvK7kSz7ZUMa/SZohadjd/7nnTUzBzJZp4mwvTUxi+osqezOz5ySt1sSvvsYl/VDSi5J+JelKSX+S9C137/kHby16W62Jl65/mbn5/HvsHvf295LekrRX0rls8WZNvL+u7LlL9LVOFTxvfMMPCIpv+AFBEX4gKMIPBEX4gaAIPxAU4QeCIvxAUIQfCOr/AADq1C4+dPlaAAAAAElFTkSuQmCC\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Route:{}\n", - "{\n", - " \"0\": \"0.08\",\n", - " \"1\": \"0.10\",\n", - " \"2\": \"0.09\",\n", - " \"3\": \"0.09\",\n", - " \"4\": \"0.12\",\n", - " \"5\": \"0.09\",\n", - " \"6\": \"0.10\",\n", - " \"7\": \"0.11\",\n", - " \"8\": \"0.09\",\n", - " \"9\": \"0.12\"\n", - "}\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "utils.predict_rest_mnist(mnist,\"mnist-classifier\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start a Load Test" + ] + }, { "cell_type": "code", - "execution_count": 31, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "node \"minikube\" labeled\r\n" - ] - } - ], + "outputs": [], "source": [ "!kubectl label nodes $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') role=locust" ] }, { "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME: loadtest\n", - "LAST DEPLOYED: Tue Jun 19 18:53:23 2018\n", - "NAMESPACE: seldon\n", - "STATUS: DEPLOYED\n", - "\n", - "RESOURCES:\n", - "==> v1/Pod(related)\n", - "NAME READY STATUS RESTARTS AGE\n", - "locust-slave-1-jmjbs 0/2 Init:0/1 0 0s\n", - "locust-master-1-6xhvd 0/2 Init:0/1 0 0s\n", - "\n", - "==> v1/ReplicationController\n", - "NAME DESIRED CURRENT READY AGE\n", - "locust-slave-1 1 1 0 0s\n", - "locust-master-1 1 1 0 0s\n", - "\n", - "==> v1/Service\n", - "NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE\n", - "locust-master-1 NodePort 10.102.138.69 5557:31388/TCP,5558:32352/TCP,8089:30904/TCP 0s\n", - "\n", - "\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!helm install seldon-core-loadtesting --name loadtest \\\n", " --namespace seldon \\\n", @@ -559,6 +365,13 @@ " --set data.size=784\n" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Version 1 Istio Routing" + ] + }, { "cell_type": "code", "execution_count": null, @@ -577,6 +390,22 @@ "!istioctl create -f istio_canary_v1.yaml" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Launch Version 2 of Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize mnist_v2.json" + ] + }, { "cell_type": "code", "execution_count": null, @@ -586,30 +415,52 @@ "!kubectl apply -f mnist_v2.json" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "** Wait until new pods are running **" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "for i in range (100):\n", - " utils.predict_rest_mnist(mnist,\"mnist-classifier\")" + "!kubectl get pods" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Updated config virtual-service/seldon/mnist-deployment to revision 3005\r\n", - "Updated config destination-rule/seldon/mnist-deployment to revision 3006\r\n" - ] - } - ], + "outputs": [], + "source": [ + "utils.predict_rest_mnist(mnist,\"mnist-classifier\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Setup Canary Routing" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pygmentize istio_canary_v2.yaml" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!istioctl replace -f istio_canary_v2.yaml" ] @@ -618,24 +469,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tear Down" + "# Setup Routing to Version 2" ] }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "release \"loadtest\" deleted\r\n" - ] - } - ], + "outputs": [], "source": [ - "!helm delete loadtest --purge" + "!pygmentize istio_canary_v3.yaml" ] }, { @@ -643,7 +486,9 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "!istioctl replace -f istio_canary_v3.yaml" + ] } ], "metadata": { diff --git a/examples/istio/canary_update/istio_canary_v2.yaml b/examples/istio/canary_update/istio_canary_v2.yaml index 0be2b8818f..f0f74e5eea 100644 --- a/examples/istio/canary_update/istio_canary_v2.yaml +++ b/examples/istio/canary_update/istio_canary_v2.yaml @@ -11,11 +11,11 @@ spec: - destination: host: mnist-deployment subset: v1 - weight: 90 + weight: 80 - destination: host: mnist-deployment subset: v2 - weight: 10 + weight: 20 --- apiVersion: networking.istio.io/v1alpha3 kind: DestinationRule diff --git a/helm-charts/seldon-core-crd/Chart.yaml b/helm-charts/seldon-core-crd/Chart.yaml index 0874dff1a7..18c55aa36a 100644 --- a/helm-charts/seldon-core-crd/Chart.yaml +++ b/helm-charts/seldon-core-crd/Chart.yaml @@ -6,4 +6,4 @@ keywords: name: seldon-core-crd sources: - https://github.com/SeldonIO/seldon-core -version: 0.1.8-SNAPSHOT +version: 0.1.8-ISTIO-SNAPSHOT diff --git a/helm-charts/seldon-core/Chart.yaml b/helm-charts/seldon-core/Chart.yaml index 22b5a32e85..d093b015e9 100644 --- a/helm-charts/seldon-core/Chart.yaml +++ b/helm-charts/seldon-core/Chart.yaml @@ -6,4 +6,4 @@ keywords: name: seldon-core sources: - https://github.com/SeldonIO/seldon-core -version: 0.1.8-SNAPSHOT +version: 0.1.8-ISTIO-SNAPSHOT diff --git a/helm-charts/seldon-core/values.yaml b/helm-charts/seldon-core/values.yaml index 90dadeebfe..814ad65784 100644 --- a/helm-charts/seldon-core/values.yaml +++ b/helm-charts/seldon-core/values.yaml @@ -2,17 +2,17 @@ apife: enabled: true image: pull_policy: IfNotPresent - tag: 0.1.8-SNAPSHOT + tag: 0.1.8-ISTIO-SNAPSHOT apife_service_type: NodePort cluster_manager: image: pull_policy: IfNotPresent - tag: 0.1.8-SNAPSHOT + tag: 0.1.8-ISTIO-SNAPSHOT java_opts: '' spring_opts: '' engine: image: - tag: 0.1.8-SNAPSHOT + tag: 0.1.8-ISTIO-SNAPSHOT rbac: enabled: true redis: diff --git a/seldon-core/seldon-core/prototypes/core.jsonnet b/seldon-core/seldon-core/prototypes/core.jsonnet index fd25b5853e..d0a28a4aec 100644 --- a/seldon-core/seldon-core/prototypes/core.jsonnet +++ b/seldon-core/seldon-core/prototypes/core.jsonnet @@ -6,12 +6,12 @@ // @optionalParam namespace string default Namespace // @optionalParam withRbac string false Whether to include RBAC setup // @optionalParam withApife string true Whether to include builtin API Oauth fornt end server for ingress -// @optionalParam apifeImage string seldonio/apife:0.1.6 Default image for API Front End +// @optionalParam apifeImage string seldonio/apife:0.1.8-ISTIO-SNAPSHOT Default image for API Front End // @optionalParam apifeServiceType string NodePort API Front End Service Type -// @optionalParam operatorImage string seldonio/cluster-manager:0.1.6 Seldon cluster manager image version +// @optionalParam operatorImage string seldonio/cluster-manager:0.1.8-ISTIO-SNAPSHOT Seldon cluster manager image version // @optionalParam operatorSpringOpts string null cluster manager spring opts // @optionalParam operatorJavaOpts string null cluster manager java opts -// @optionalParam engineImage string seldonio/engine:0.1.6 Seldon engine image version +// @optionalParam engineImage string seldonio/engine:0.1.8-ISTIO-SNAPSHOT Seldon engine image version // TODO(https://github.com/ksonnet/ksonnet/issues/222): We have to add namespace as an explicit parameter // because ksonnet doesn't support inheriting it from the environment yet. From bf9453349cf584ec3dbc52a41d27d43bddbabd3d Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Thu, 21 Jun 2018 07:44:33 +0100 Subject: [PATCH 16/20] add initial istio docs --- docs/istio.md | 8 ++++++++ readme.md | 2 ++ 2 files changed, 10 insertions(+) create mode 100644 docs/istio.md diff --git a/docs/istio.md b/docs/istio.md new file mode 100644 index 0000000000..470f55b11d --- /dev/null +++ b/docs/istio.md @@ -0,0 +1,8 @@ +# Istio and Seldon + +[Istio](https://istio.io/) provides service mesh functionality and can be a useful addition to Seldon to provide extra traffic management, end-to-end security and policy enforcement in your runtime machine learning deployment graph. Seldon-core can be seen as providing a service graph for machine learning deployments. As part of that it provdes an Operator which takes your ML deployment graph definition described as a SeldonDeployment kubernetes resource and deploys and manages it on a kubernetes cluster so you can connect your business applications that need to access machine learning services. Data scientists can focus on building pluggable docker containers for parts of their runtime machine learning graph, such as runtime inference, transformations, outlier detection, ensemblers etc. These can be composed together as needed to satisfy your runtime ML functionality. To allow modules to be built without knowing what service graph they will exist in means Seldon also deploys a Service Orchestrator as part of each deployment which manages the request/reponse flow to satisfy the defined ML service graph for multi-component graphs. + +Out of the box Seldon provides rolling updates to SeldonDeployment service graphs provided by the underlying kubernetes functionality. However, there are cases where you want to manage updates to your ML deployments in a more controlled way with fine grained traffic management including canary updates, blue-green deployments and shadowing. This is where Istio can help in combination with Seldon. + +[An example step-by-step guide to canary deployemts using Istio and Seldon is provided](../examples/istio/canary_update/canary.ipynb). + diff --git a/readme.md b/readme.md index 197aa5c6e1..3998639f1f 100644 --- a/readme.md +++ b/readme.md @@ -89,6 +89,8 @@ Seldon-core allows various types of components to be built and plugged into the * Seldon-core can be used to [serve deep learning models trained using FfDL](https://github.com/IBM/FfDL/blob/master/community/FfDL-Seldon/README.md). * [Train and deploy a Tensorflow MNIST classififer using FfDL and Seldon.](https://github.com/IBM/FfDL/blob/master/community/FfDL-Seldon/tf-model/README.md) * [Train and deploy a PyTorch MNIST classififer using FfDL and Seldon.](https://github.com/IBM/FfDL/blob/master/community/FfDL-Seldon/pytorch-model/README.md) + * [Istio and Seldon](./docs/istio.md) + * [Canary deployemts using Istio and Seldon.](../examples/istio/canary_update/canary.ipynb). ## Install From 0e639d2b796c7280cce837b0d6920b1b3e39fb8b Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Thu, 21 Jun 2018 08:11:49 +0100 Subject: [PATCH 17/20] add v1alpha2 update docs --- docs/v1alpha2_update.md | 123 ++++++++++++++++++++++++++++++++++++++++ readme.md | 11 ++++ 2 files changed, 134 insertions(+) create mode 100644 docs/v1alpha2_update.md diff --git a/docs/v1alpha2_update.md b/docs/v1alpha2_update.md new file mode 100644 index 0000000000..87b5cba37d --- /dev/null +++ b/docs/v1alpha2_update.md @@ -0,0 +1,123 @@ +# V1Alpha2 Update + + * The ```PredictorSpec componentSpec``` is now ```componentSpecs``` and takes a list of ```PodTemplateSpecs``` allowing you to split your runtime graph into separate Kubernetes Deployments as needed. See the new [proto definiton](./proto/seldon_deployment.proto). To update existing resources: + * Change ``` "apiVersion": "machinelearning.seldon.io/v1alpha1"``` to ``` "apiVersion": "machinelearning.seldon.io/v1alpha2"``` + * Change ```componentSpec``` -> ```componentSpecs``` and enclose the existing ```PodTemplateSpec``` in a single element list + +For example change: + +``` +{ + "apiVersion": "machinelearning.seldon.io/v1alpha1", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "seldon-deployment-example" + }, + "spec": { + "annotations": { + "project_name": "FX Market Prediction", + "deployment_version": "v1" + }, + "name": "test-deployment", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpec": { + "spec": { + "containers": [ + { + "image": "seldonio/mock_classifier:1.0", + "imagePullPolicy": "IfNotPresent", + "name": "classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }, + "graph": { + "children": [], + "name": "classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "fx-market-predictor", + "replicas": 1, + "annotations": { + "predictor_version" : "v1" + } + } + ] + } +} + +``` + +to + +``` +{ + "apiVersion": "machinelearning.seldon.io/v1alpha2", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "seldon-deployment-example" + }, + "spec": { + "annotations": { + "project_name": "FX Market Prediction", + "deployment_version": "v1" + }, + "name": "test-deployment", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpecs": [{ + "spec": { + "containers": [ + { + "image": "seldonio/mock_classifier:1.0", + "imagePullPolicy": "IfNotPresent", + "name": "classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "graph": { + "children": [], + "name": "classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "fx-market-predictor", + "replicas": 1, + "annotations": { + "predictor_version" : "v1" + } + } + ] + } +} + +``` \ No newline at end of file diff --git a/readme.md b/readme.md index 3998639f1f..62db935071 100644 --- a/readme.md +++ b/readme.md @@ -45,6 +45,17 @@ Machine learning deployment has many [challenges](./docs/challenges.md). Seldon A [Kubernetes](https://kubernetes.io/) Cluster. Kubernetes can be deployed into many environments, both in cloud and on-premise. +## Important: V1Alpha2 Update + + **We have updated our core API to v1alpha2 which has a breaking change from v1alpha1 in the SeldonDeployments CRD** + +[Read details of how to update your kubernetes SeldonDeployment resources](./docs/v1alpha2_update.md). + + * **0.2** releases will now respect the v1alpha2 API. + * **0.1** releases respect the v1alpha1 API and will not be worked on further. + +It is possible to deploy Seldon with two operators that can handle both v1alpha1 resouces and v1alpha2 resources though this is not part of our standard deployment docs. If you need this please get in touch. + ## Quick Start - Jupyter notebooks showing worked examples: From 705074a61021594735de7b94c9cdeeb400836287 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Thu, 21 Jun 2018 11:15:45 +0100 Subject: [PATCH 18/20] add hash if deployment name too long --- .../k8s/SeldonDeploymentOperatorImpl.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java index 1feb3d58cb..3d4fd0afda 100644 --- a/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java +++ b/cluster-manager/src/main/java/io/seldon/clustermanager/k8s/SeldonDeploymentOperatorImpl.java @@ -21,10 +21,10 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.StringJoiner; +import org.apache.commons.codec.digest.DigestUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -326,9 +326,17 @@ private String getPredictorServiceNameKey(String containerName) return LABEL_SELDON_APP+"-"+containerName; } + private String hash(String key) { + return DigestUtils.md5Hex(key).toLowerCase(); + } + @Override public String getSeldonServiceName(SeldonDeployment dep,PredictorSpec pred,String key) { - return dep.getSpec().getName() + "-" + pred.getName()+"-"+key; + String svcName = dep.getSpec().getName() + "-" + pred.getName()+"-"+key; + if (svcName.length() > 63) + return "seldon-"+hash(svcName); + else + return svcName; } From f29705da6c43ebca382c7be45867ea7618d82905 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 23 Jun 2018 09:07:38 +0100 Subject: [PATCH 19/20] updated docs --- docs/crd/readme.md | 11 +- docs/getting_started/minikube.md | 216 ---------------------------- docs/getting_started/readme.md | 37 ++++- docs/reference/seldon-deployment.md | 5 + readme.md | 2 + 5 files changed, 48 insertions(+), 223 deletions(-) delete mode 100644 docs/getting_started/minikube.md diff --git a/docs/crd/readme.md b/docs/crd/readme.md index b2cf20ab64..65a875529a 100644 --- a/docs/crd/readme.md +++ b/docs/crd/readme.md @@ -24,9 +24,9 @@ message SeldonDeployment { } ``` -The core deployment spec consists of a set of ```predictors```. Each predictor represents a seperate runtime serving graph. The set of predictors will serve request as controlled by a load balancer. At present the share of traffic will be in relation to the number of replicas each predictor has. A use case for two predictors would be a main deployment and a canary, with the main deployment having 9 replicas and the canary 1, so the canary receives 10% of the overall traffic. Each predictor will be a seperately managed deployment with Kubernetes so it is safe to add and remove predictors without affecting existing predictors. +The core deployment spec consists of a set of ```predictors```. Each predictor represents a seperate runtime serving graph. The set of predictors will serve request as controlled by a load balancer. At present the share of traffic will be in relation to the number of replicas each predictor has. A use case for two predictors would be a main deployment and a canary, with the main deployment having 9 replicas and the canary 1, so the canary receives 10% of the overall traffic. Each predictor will be a seperately set of managed deployments with Kubernetes so it is safe to add and remove predictors without affecting existing predictors. -To allow an OAuth API to be provisioned you should specify an OAuth key and secret. +To allow an OAuth API to be provisioned you should specify an OAuth key and secret. If you are using Ambassador you will not need this as you can plug in your own external authentication using Ambassador. ```proto @@ -44,18 +44,19 @@ For each predictor you should at a minimum specify: * A unique name * A PredictiveUnit graph that presents the tree of components to deploy. - * A componentSpec which describes the set of images for parts of your container graph that will be instigated as microservice containers. These containers will have been wrapped to work within the [internal API](../reference/internal-api.md). This component spec is a standard [PodTemplateSpec](https://kubernetes.io/docs/api-reference/extensions/v1beta1/definitions/#_v1_podtemplatespec). + * One or more componentSpecs which describes the set of images for parts of your container graph that will be instigated as microservice containers. These containers will have been wrapped to work within the [internal API](../reference/internal-api.md). This component spec is a standard [PodTemplateSpec](https://kubernetes.io/docs/api-reference/extensions/v1beta1/definitions/#_v1_podtemplatespec). For complex grahs you can decide to use several componentSpecs so as to separate your components into separate Pods each with their own resource requirements. * If you leave the ports empty for each container they will be added automatically and matched to the ports in the graph specification. If you decide to specify the ports manually they should match the port specified for the matching component in the graph specification. * the number of replicas of this predictor to deploy ```proto - message PredictorSpec { required string name = 1; // A unique name not used by any other predictor in the deployment. required PredictiveUnit graph = 2; // A graph describing how the predictive units are connected together. - required k8s.io.api.core.v1.PodTemplateSpec componentSpec = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. + repeated k8s.io.api.core.v1.PodTemplateSpec componentSpecs = 3; // A description of the set of containers used by the graph. One for each microservice defined in the graph. Can be split over 1 or more PodTemplateSpecs. optional int32 replicas = 4; // The number of replicas of the predictor to create. map annotations = 5; // Arbitrary annotations. + optional k8s.io.api.core.v1.ResourceRequirements engineResources = 6; // Optional set of resources for the Seldon engine which is added to each Predictor graph to manage the request/response flow + map labels = 7; // labels to be attached to entry deplyment for this predictor } ``` diff --git a/docs/getting_started/minikube.md b/docs/getting_started/minikube.md deleted file mode 100644 index 3b931ad4ed..0000000000 --- a/docs/getting_started/minikube.md +++ /dev/null @@ -1,216 +0,0 @@ - -# Getting started on Minikube - - -In this guide, we will show how to create, deploy and serve an Iris classification model using Seldon Core running on a Minikube cluster. Seldon Core uses [helm](https://github.com/kubernetes/helm) charts to start and runs on [kubernetes](https://kubernetes.io/) clusters. Minikube is a tool that makes it easy to run Kubernetes locally, and runs a single-node Kubernetes cluster inside a virtual machine on your laptop. - - -### Prerequisites - -The following packages need to be installed on your machine. - -* [minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/) >= 0.24.0 -* [helm](https://github.com/kubernetes/helm/blob/master/docs/install.md) >= 2.7.0 - -* [sklearn](http://scikit-learn.org/stable/) - - Sklearn is needed only to train the iris classifier example below. Seldon Core doesn't not require sklearn installed on your machine to run. - - -### Before starting: run a Minikube cluster locally - -Before starting, you need to have [minikube installed](https://kubernetes.io/docs/tasks/tools/install-minikube/) on your machine. - -1. Start a Kubernetes local cluster in your machine using Minikube with RBAC enabled: - - ```bash - minikube start --memory=8000 --feature-gates=CustomResourceValidation=true --extra-config=apiserver.Authorization.Mode=RBAC - ``` - -Once the cluster is created, Minikube should automatically point your kubectl cli to the minikube cluster. - -### Start seldon-core - -You can now start Seldon Core in your minikube cluster. - - -1. Seldon Core uses helm charts to start. To use Seldon Core, you need [helm installed](https://github.com/kubernetes/helm/blob/master/docs/install.md) in your machine. To Initialize helm, type on command line: - - ```bash - helm init - ``` - -2. Seldon Core uses helm charts to start, which are stored in google storage. -Use the charts to install the CRD and then the core components. Enabling reporting of anonymous usage metrics is optional, see [Usage Reporting](/docs/developer/readme.md#usage-reporting). - - - ```bash - helm install seldon-core-crd --name seldon-core-crd \ - --repo https://storage.googleapis.com/seldon-charts \ - --set usage_metrics.enabled=true - helm install seldon-core --name seldon-core \ - --repo https://storage.googleapis.com/seldon-charts - ``` - -Seldon Core should now be running on your cluster. You can verify if all the pods are up and running typing on command line ```helm status seldon-core``` or ```kubectl get pods``` - -### Wrap your model - -In this session, we show how to wrap the sklearn iris classifier in the [seldon-core](https://github.com/SeldonIO/seldon-core) repository using Seldon Core python wrappers. The example consists of a logistic regression model trained on the [Iris dataset](http://scikit-learn.org/stable/auto_examples/datasets/plot_iris_dataset.html). - -1. Clone the seldon-core repository: - - ```bash - git clone https://github.com/SeldonIO/seldon-core - ``` - -2. Train and save the sklearn iris classifier example model using the provided script ```train_iris.py```: - - ```bash - cd seldon-core/examples/models/sklearn_iris/ - ``` - ```bash - python train_iris.py - ```` - - This will train a simple logistic regression model on the iris dataset and save the model in the same folder. - - -3. Wrap your saved model using the core-python-wrapper docker image: - ```bash - docker run -v $(pwd):/model seldonio/core-python-wrapper:0.7 /model IrisClassifier 0.1 seldonio --force - ``` - -4. Build the docker image locally - ```bash - cd build - ./build_image.sh - ``` - This will create the docker image ```seldonio/irisclassifier:0.1``` inside the minikube cluster which is ready for deployment with Seldon Core. - - -### Deploy your model - -The docker image version of your model is deployed through a json configuration file. A general template for the configuration can be found [here](https://github.com/SeldonIO/seldon-core/blob/master/examples/models/sklearn_iris/sklearn_iris_deployment.json). For the sklearn iris example, we have already created a deployment file ```sklearn_iris_deployment.json``` in the ```sklearn_iris``` folder: - -```json -{ - "apiVersion": "machinelearning.seldon.io/v1alpha2", - "kind": "SeldonDeployment", - "metadata": { - "labels": { - "app": "seldon" - }, - "name": "seldon-deployment-example" - }, - "spec": { - "annotations": { - "project_name": "Iris classification", - "deployment_version": "0.1" - }, - "name": "sklearn-iris-deployment", - "oauth_key": "oauth-key", - "oauth_secret": "oauth-secret", - "predictors": [ - { - "componentSpecs": [{ - "spec": { - "containers": [ - { - "image": "seldonio/irisclassifier:0.1", - "imagePullPolicy": "IfNotPresent", - "name": "sklearn-iris-classifier", - "resources": { - "requests": { - "memory": "1Mi" - } - } - } - ], - "terminationGracePeriodSeconds": 20 - } - }], - "graph": { - "children": [], - "name": "sklearn-iris-classifier", - "endpoint": { - "type" : "REST" - }, - "type": "MODEL" - }, - "name": "sklearn-iris-predictor", - "replicas": 1, - "annotations": { - "predictor_version" : "0.1" - } - } - ] - } -} -``` - -1. To deploy the model in seldon core, type on command line (from the that contains the sklearn_iris_deployment.json file): - - ```bash - kubectl apply -f sklearn_iris_deployment.json - ``` - -2. The deployment will take a few seconds to be ready. To check if your deployment is ready: - - ```bash - kubectl describe seldondeployments seldon-deployment-example - ``` - - - -### Serve your model: - -In order to send a prediction request to your model, you need to query the Seldon Core api server and obtain an authentication token from your model key and secret (in this example key and secret are set to "oauth-key" and "oauth-secret" for simplicity). The api server is running on minikube. To query your model you need to - -1. Set the api server IP and port: - - ```bash - SERVER=$(minikube ip):$(kubectl get svc -l app=seldon-apiserver-container-app -o jsonpath='{.items[0].spec.ports[0].nodePort}') - ``` - -2. Get the authorization token using your key ("oauth-key" here) and secret ("oauth-secret"): - - ```bash - TOKEN=`curl -s -H "Accept: application/json" \ - oauth-key:oauth-secret@${SERVER}/oauth/token -d grant_type=client_credentials | jq -r '.access_token'` - ```` - -3. Query the api server prediction endpoint. The json object at the end is your message containing the values for your features: - ```bash - curl -s -H "Content-Type:application/json" -H "Accept: application/json" \ - -H "Authorization: Bearer $TOKEN" ${SERVER}/api/v0.1/predictions -d \ - '{"meta":{},"data":{"names":["sepal length (cm)","sepal width (cm)", "petal length (cm)","petal width (cm)"],"ndarray":[[5.1,3.5,1.4,0.2]]}}' - -The response from the server should be a json object of this type: - -```json -{ - "meta": { - "puid": "lhq41l3q736q7tnrij8o3lod8u", - "tags": { - }, - "routing": { - } - }, - "data": { - "names": ["t:0", "t:1", "t:2"], - "ndarray": [[0.8796816489561845, 0.12030753790659003, 1.0813137225507727E-5]] - } -} -``` - -The response contains: - -* a "meta" dictionary: This dictionary contains various metadata: - * "puid": A unique identifier for the prediction. - * "tags": Optional tags. Empty in this case. - * "routing": This field is relevant when the deployment contain a more complex graph. In this case is empty since we are deploying a single model. -* "data" dictionary: This dictionary contains the predictions for your model classes - * "names": The names of your classes. - * "ndarray": The predicted probabilities for each class. - - diff --git a/docs/getting_started/readme.md b/docs/getting_started/readme.md index dd83c972e2..195204604e 100644 --- a/docs/getting_started/readme.md +++ b/docs/getting_started/readme.md @@ -1,6 +1,39 @@ -# Getting Started +# Getting started Seldon Core - - [Quick Start using Minikube](./minikube.md) +There are 4 steps to using seldon-core. + * Install seldon-core onto a kubernetes cluster + * Wrap your components (usually runtime model servers) as Docker containers that respect the internal Seldon microservice API. + * Define your runtime service graph as a SeldonDeployment resource + * Deploy your model and serve predictions +[IMAGE steps] + +# Install Seldon Core + +To install seldon-core follow the [installation guide](../install.md). + +# Wrap Your Model + +The components you want to run in production need to be wrapped as Docker containers that respect the [Seldon microservice API](../reference/internal-api.md). You can create models that serve predictions, routers that decide on where requests go, such as A-B Tests, Combiners that combine responses and transformers that provide generic components that can transform requests and/or responses. + +To allow users to easily wrap machine learning components built using different langauges and toolkits we provide wrappers that allow you easily to build a docker container from your code that can be run inside seldon-core. Our current recommended tool is RedHat's Source-to-Image. Wrapping your models is discussed [here](../wrappers/readme.md). + +# Define Runtime Service Graph + +To run your machine learning graph on Kubernetes you need to define how the components you created in the last step fit together to represent a service graph. This is defined inside a [SeldonDeployment Kubernetes Custom resource](../reference/seldon-deployment.md). A [guide to constructing this custom resource service graph is provided](../crd/readme.md). + + +[IMAGE graph] + +# Deploy and Serve Predictions + +You can use ```kubectl``` to deploy your ML service like any other Kubernetes resource. This is discussed [here](../deploying.md). + +# Worked Examples + + * [Jupyter notebooks showing worked examples](../../readme.md#quick-start) + * [Integration with other machine learning frameworks](../../readme.md#integrations) + +# Production \ No newline at end of file diff --git a/docs/reference/seldon-deployment.md b/docs/reference/seldon-deployment.md index 0c828581c5..c23699a563 100644 --- a/docs/reference/seldon-deployment.md +++ b/docs/reference/seldon-deployment.md @@ -149,6 +149,11 @@ message Parameter { ## Single Model + * The model is contained in the image ```seldonio/mock_classifier:1.0``` + * The model requests 1 MB of memory + * The model defines oauth key and secret for use with seldon-core's built in API gateway. + * The model supports a REST API + ```json { "apiVersion": "machinelearning.seldon.io/v1alpha2", diff --git a/readme.md b/readme.md index 62db935071..34f33965a5 100644 --- a/readme.md +++ b/readme.md @@ -58,6 +58,8 @@ It is possible to deploy Seldon with two operators that can handle both v1alpha1 ## Quick Start +Read the [overview to using seldon-core](./docs/gettings_started/readme.md).. + - Jupyter notebooks showing worked examples: * Minikube: * [Jupyter Notebook showing deployment of prebuilt model using Minikube - with RBAC](https://github.com/SeldonIO/seldon-core/blob/master/notebooks/kubectl_demo_minikube_rbac.ipynb) From 02aec4a840143eb5a1747723af0886573a111b47 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Sat, 23 Jun 2018 09:14:46 +0100 Subject: [PATCH 20/20] extra model examples --- .gitignore | 1 + .../resources/model_multiple_versions.json | 79 ++++++++++++++++ notebooks/resources/random_ab_test_1pod.json | 89 +++++++++++++++++++ 3 files changed, 169 insertions(+) create mode 100644 notebooks/resources/model_multiple_versions.json create mode 100644 notebooks/resources/random_ab_test_1pod.json diff --git a/.gitignore b/.gitignore index a5d0b2aab8..a36f21bab4 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,4 @@ examples/models/pyspark_pmml/derby.log examples/models/pyspark_pmml/mnist_train.csv examples/models/pyspark_pmml/src/main/resources/model.pmml examples/models/pyspark_pmml/.gitignore +examples/models/h2o_mojo/src/main/resources/model.zip diff --git a/notebooks/resources/model_multiple_versions.json b/notebooks/resources/model_multiple_versions.json new file mode 100644 index 0000000000..cf881c7242 --- /dev/null +++ b/notebooks/resources/model_multiple_versions.json @@ -0,0 +1,79 @@ +{ + "apiVersion": "machinelearning.seldon.io/v1alpha2", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "seldon-deployment-example" + }, + "spec": { + "annotations": { + "project_name": "FX Market Prediction", + "deployment_version": "v1" + }, + "name": "test-deployment", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpecs": [{ + "metadata":{ + "labels":{ + "version":"v1" + } + }, + "spec": { + "containers": [ + { + "image": "seldonio/mock_classifier:1.0", + "imagePullPolicy": "IfNotPresent", + "name": "classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + },{ + "metadata":{ + "labels":{ + "version":"v2" + } + }, + "spec": { + "containers": [ + { + "image": "seldonio/mean_classifier:0.6", + "imagePullPolicy": "IfNotPresent", + "name": "classifier", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "graph": { + "children": [], + "name": "classifier", + "endpoint": { + "type" : "REST" + }, + "type": "MODEL" + }, + "name": "fx-market-predictor", + "replicas": 1, + "annotations": { + "predictor_version" : "v1" + } + } + ] + } +} diff --git a/notebooks/resources/random_ab_test_1pod.json b/notebooks/resources/random_ab_test_1pod.json new file mode 100644 index 0000000000..342204b49a --- /dev/null +++ b/notebooks/resources/random_ab_test_1pod.json @@ -0,0 +1,89 @@ +{ + "apiVersion": "machinelearning.seldon.io/v1alpha2", + "kind": "SeldonDeployment", + "metadata": { + "labels": { + "app": "seldon" + }, + "name": "seldon-deployment-example" + }, + "spec": { + "annotations": { + "project_name": "FX Market Prediction", + "deployment_version": "v1" + }, + "name": "test-deployment-abtest", + "oauth_key": "oauth-key", + "oauth_secret": "oauth-secret", + "predictors": [ + { + "componentSpecs": [{ + "spec": { + "containers": [ + { + "image": "seldonio/mock_classifier:1.0", + "imagePullPolicy": "IfNotPresent", + "name": "classifier-1", + "resources": { + "requests": { + "memory": "1Mi" + } + } + }, + { + "image": "seldonio/mock_classifier:1.0", + "imagePullPolicy": "IfNotPresent", + "name": "classifier-2", + "resources": { + "requests": { + "memory": "1Mi" + } + } + } + ], + "terminationGracePeriodSeconds": 20 + } + }], + "name": "fx-market-predictor", + "replicas": 1, + "annotations": { + "predictor_version": "v1" + }, + "graph": { + "name": "random-ab-test", + "endpoint":{}, + "implementation":"RANDOM_ABTEST", + "parameters": [ + { + "name":"ratioA", + "value":"0.5", + "type":"FLOAT" + } + ], + "children": [ + { + "name": "classifier-1", + "endpoint":{ + "type":"REST" + }, + "type":"MODEL", + "children":[] + }, + { + "name": "classifier-2", + "endpoint":{ + "type":"REST" + }, + "type":"MODEL", + "children":[] + } + ] + } + } + ] + } +} + + + +