diff --git a/.lighthouse/jenkins-x/README.md b/.lighthouse/jenkins-x/README.md new file mode 100644 index 0000000000..7c95d8c78e --- /dev/null +++ b/.lighthouse/jenkins-x/README.md @@ -0,0 +1,56 @@ +# Overview + +The setup of the nodes include the following + +# General Node Pool + +This is a node pool that is used for general processing, including the release build, the integration tests, etc. + +There is a benchmarking node pool with the following requirements: +* taints: job-type=benchmark:NoSchedule + +The command used to create it was the following: + +``` +gcloud container node-pools create general-pipelines-pool --zone=us-central1-a --cluster=tf-jx-working-weevil --node-taints=job-type=general:NoSchedule --enable-autoscaling --max-nodes=3 --min-nodes=0 --num-nodes=0 --machine-type=e2-standard-8 --disk-size=1000GB +``` + +It is possible to create pipelines that reference this job by using: + +``` + nodeSelector: + cloud.google.com/gke-nodepool: general-pipelines-pool + tolerations: + - key: job-type + operator: Equals + value: general + effect: NoSchedule +``` + + + +# Benchmark Node Pool + +This is the node pool that is used specifically for benchmarking tasks, where only 1 benchmark task would fit a single node. + +There is a benchmarking node pool with the following requirements: +* taints: job-type=benchmark:NoSchedule + +The command used to create it was the following: + +``` +gcloud container node-pools create benchmark-pipelines-pool --zone=us-central1-a --cluster=tf-jx-working-weevil --node-taints=job-type=benchmark:NoSchedule --enable-autoscaling --max-nodes=1 --min-nodes=0 --num-nodes=0 --machine-type=e2-standard-8 --disk-size=1000GB +``` + +It is possible to create pipelines that reference this job by using: + +``` + nodeSelector: + cloud.google.com/gke-nodepool: benchmark-pipelines-pool + tolerations: + - key: job-type + operator: Equals + value: benchmark + effect: NoSchedule +``` + diff --git a/.lighthouse/jenkins-x/benchmark.yaml b/.lighthouse/jenkins-x/benchmark.yaml new file mode 100644 index 0000000000..ea7e6c03c0 --- /dev/null +++ b/.lighthouse/jenkins-x/benchmark.yaml @@ -0,0 +1,70 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + creationTimestamp: null + name: benchmark +spec: + pipelineSpec: + tasks: + - name: benchmark-test-task + taskSpec: + stepTemplate: + name: "" + workingDir: /workspace/source + steps: + - image: uses:jenkins-x/jx3-pipeline-catalog/tasks/git-clone/git-clone-pr.yaml@versionStream + name: "" + - name: benchmark-step + image: seldonio/core-builder:0.21 + env: + - name: SELDON_E2E_TESTS_TO_RUN + value: benchmark + - name: SELDON_E2E_TESTS_POD_INFORMATION + value: "true" + command: + - bash + - -c + - cd testing/scripts && bash kind_test_all.sh + volumeMounts: + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - name: dind-storage + mountPath: /var/lib/docker + resources: + requests: + cpu: 4 + memory: 10000Mi + ephemeral-storage: "150Gi" + limits: + cpu: 4 + memory: 10000Mi + ephemeral-storage: "150Gi" + securityContext: + privileged: true + imagePullPolicy: Always + volumes: + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: dind-storage + emptyDir: {} + podTemplate: + nodeSelector: + cloud.google.com/gke-nodepool: benchmark-pipelines-pool + tolerations: + - key: job-type + operator: Equal + value: benchmark + effect: NoSchedule + serviceAccountName: tekton-bot + timeout: 6h0m0s +status: {} + diff --git a/.lighthouse/jenkins-x/integration.yaml b/.lighthouse/jenkins-x/integration.yaml index 26b53fa7cb..35f46dd9a1 100644 --- a/.lighthouse/jenkins-x/integration.yaml +++ b/.lighthouse/jenkins-x/integration.yaml @@ -37,11 +37,11 @@ spec: requests: cpu: 3 memory: 8000Mi - ephemeral-storage: "100Gi" + ephemeral-storage: "150Gi" limits: cpu: 3 memory: 8000Mi - ephemeral-storage: "100Gi" + ephemeral-storage: "150Gi" securityContext: privileged: true imagePullPolicy: Always @@ -56,7 +56,14 @@ spec: type: Directory - name: dind-storage emptyDir: {} - podTemplate: {} + podTemplate: + nodeSelector: + cloud.google.com/gke-nodepool: general-pipelines-pool + tolerations: + - key: job-type + operator: Equal + value: general + effect: NoSchedule serviceAccountName: tekton-bot timeout: 6h0m0s status: {} diff --git a/.lighthouse/jenkins-x/notebooks.yaml b/.lighthouse/jenkins-x/notebooks.yaml index fc1be05810..b7fb9d2ffe 100644 --- a/.lighthouse/jenkins-x/notebooks.yaml +++ b/.lighthouse/jenkins-x/notebooks.yaml @@ -56,7 +56,14 @@ spec: type: Directory - name: dind-storage emptyDir: {} - podTemplate: {} + podTemplate: + nodeSelector: + cloud.google.com/gke-nodepool: general-pipelines-pool + tolerations: + - key: job-type + operator: Equal + value: general + effect: NoSchedule serviceAccountName: tekton-bot timeout: 6h0m0s status: {} diff --git a/.lighthouse/jenkins-x/release.yaml b/.lighthouse/jenkins-x/release.yaml index 4d2ff35e83..eada48839b 100644 --- a/.lighthouse/jenkins-x/release.yaml +++ b/.lighthouse/jenkins-x/release.yaml @@ -60,7 +60,14 @@ spec: path: config.json secretName: jenkins-docker-cfg - podTemplate: {} + podTemplate: + nodeSelector: + cloud.google.com/gke-nodepool: general-pipelines-pool + tolerations: + - key: job-type + operator: Equal + value: general + effect: NoSchedule serviceAccountName: tekton-bot timeout: 6h0m0s status: {} diff --git a/.lighthouse/jenkins-x/triggers.yaml b/.lighthouse/jenkins-x/triggers.yaml index 566848dbf2..572d0dd89d 100644 --- a/.lighthouse/jenkins-x/triggers.yaml +++ b/.lighthouse/jenkins-x/triggers.yaml @@ -20,9 +20,16 @@ spec: context: "release-build-push" always_run: false optional: false - trigger: (?m)^/test( all| release.*),?(s+|$) + trigger: (?m)^/test( release.*),?(s+|$) rerun_command: "/test release" source: "release.yaml" + - name: benchmark + context: "benchmark" + always_run: false + optional: false + trigger: (?m)^/test( benchmark.*),?(s+|$) + rerun_command: "/test benchmark" + source: "benchmark.yaml" postsubmits: - name: release context: "release" diff --git a/ci_build_and_push_images.sh b/ci_build_and_push_images.sh index b468a9c550..05a0939b42 100755 --- a/ci_build_and_push_images.sh +++ b/ci_build_and_push_images.sh @@ -71,7 +71,15 @@ function build_push_mock { make \ -C examples/models/mean_classifier \ build \ - push + push && \ + make \ + -C testing/docker/echo-model \ + build_image \ + push_image && \ + make \ + -C testing/docker/fixed-model \ + build_images \ + push_images MOCK_MODEL_EXIT_VALUE=$? } diff --git a/core-builder/Dockerfile b/core-builder/Dockerfile index 68bfb58ae8..3007b52062 100644 --- a/core-builder/Dockerfile +++ b/core-builder/Dockerfile @@ -133,6 +133,19 @@ RUN go get -u github.com/onsi/ginkgo/ginkgo # Helm docs RUN GO111MODULE=on go get github.com/norwoodj/helm-docs/cmd/helm-docs@f66fdbd6fe +# Argo workflows CLI +RUN wget https://github.com/argoproj/argo-workflows/releases/download/v3.0.8/argo-linux-amd64.gz && \ + gunzip argo-linux-amd64.gz && \ + mv argo-linux-amd64 argo && \ + chmod a+x argo && \ + mv argo /usr/local/bin/argo + +# Installing jx +RUN wget https://github.com/jenkins-x/jx-cli/releases/download/v3.1.242/jx-cli-linux-amd64.tar.gz && \ + tar -zxvf jx-cli-linux-amd64.tar.gz && \ + chmod a+x jx && \ + mv jx /usr/local/bin/jx + WORKDIR /work # Define default command. diff --git a/core-builder/Makefile b/core-builder/Makefile index dbe2529db3..21a0a3060b 100644 --- a/core-builder/Makefile +++ b/core-builder/Makefile @@ -1,5 +1,5 @@ DOCKER_IMAGE_NAME=seldonio/core-builder -DOCKER_IMAGE_VERSION=0.20 +DOCKER_IMAGE_VERSION=0.21 build_docker_image: cp ../testing/scripts/dev_requirements.txt . diff --git a/doc/source/examples/vegeta_bench_argo_workflows.nblink b/doc/source/examples/vegeta_bench_argo_workflows.nblink index e549047ce5..5d4670ee36 100644 --- a/doc/source/examples/vegeta_bench_argo_workflows.nblink +++ b/doc/source/examples/vegeta_bench_argo_workflows.nblink @@ -1,3 +1,3 @@ { - "path": "../../../examples/batch/benchmarking-argo-workflows/README.ipynb" + "path": "../../../testing/benchmarking/automated-benchmark/README.ipynb" } diff --git a/examples/batch/benchmarking-argo-workflows/README.ipynb b/examples/batch/benchmarking-argo-workflows/README.ipynb deleted file mode 100644 index 50614848d1..0000000000 --- a/examples/batch/benchmarking-argo-workflows/README.ipynb +++ /dev/null @@ -1,2452 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Benchmarking with Argo Worfklows & Vegeta\n", - "\n", - "In this notebook we will dive into how you can run bench marking with batch processing with Argo Workflows, Seldon Core and Vegeta.\n", - "\n", - "Dependencies:\n", - "\n", - "* Seldon core installed as per the docs with Istio as an ingress \n", - "* Argo Workfklows installed in cluster (and argo CLI for commands)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup\n", - "\n", - "### Install Seldon Core\n", - "Use the notebook to [set-up Seldon Core with Ambassador or Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html).\n", - "\n", - "Note: If running with KIND you need to make sure do follow [these steps](https://github.com/argoproj/argo/issues/2376#issuecomment-595593237) as workaround to the `/.../docker.sock` known issue.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Install Argo Workflows\n", - "You can follow the instructions from the official [Argo Workflows Documentation](https://github.com/argoproj/argo#quickstart).\n", - "\n", - "You also need to make sure that argo has permissions to create seldon deployments - for this you can just create a default-admin rolebinding as follows:" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "rolebinding.rbac.authorization.k8s.io/default-admin created\r\n" - ] - } - ], - "source": [ - "!kubectl create rolebinding default-admin --clusterrole=admin --serviceaccount=default:default" - ] - }, - { - "cell_type": "code", - "execution_count": 117, - "metadata": {}, - "outputs": [], - "source": [ - "def get_results(results, print_results=True):\n", - " final = {}\n", - " if \"average\" in results:\n", - " final[\"mean\"] = results[\"average\"] / 1e6\n", - " if results.get(\"latencyDistribution\", False):\n", - " final[\"50th\"] = results[\"latencyDistribution\"][-5][\"latency\"] / 1e6\n", - " final[\"90th\"] = results[\"latencyDistribution\"][-3][\"latency\"] / 1e6\n", - " final[\"95th\"] = results[\"latencyDistribution\"][-2][\"latency\"] / 1e6\n", - " final[\"99th\"] = results[\"latencyDistribution\"][-1][\"latency\"] / 1e6\n", - " final[\"rate\"] = results[\"rps\"]\n", - " final[\"errors\"] = results[\"statusCodeDistribution\"]\n", - " else:\n", - " final[\"mean\"] = results[\"latencies\"][\"mean\"] / 1e6\n", - " final[\"50th\"] = results[\"latencies\"][\"50th\"] / 1e6\n", - " final[\"90th\"] = results[\"latencies\"][\"90th\"] / 1e6\n", - " final[\"95th\"] = results[\"latencies\"][\"95th\"] / 1e6\n", - " final[\"99th\"] = results[\"latencies\"][\"99th\"] / 1e6\n", - " final[\"rate\"] = results[\"throughput\"]\n", - " final[\"errors\"] = results[\"errors\"]\n", - " if print_results:\n", - " print(\"Latencies:\")\n", - " print(\"\\tmean:\", final[\"mean\"], \"ms\")\n", - " print(\"\\t50th:\", final[\"50th\"], \"ms\")\n", - " print(\"\\t90th:\", final[\"90th\"], \"ms\")\n", - " print(\"\\t95th:\", final[\"95th\"], \"ms\")\n", - " print(\"\\t99th:\", final[\"99th\"], \"ms\")\n", - " print(\"\")\n", - " print(\"Rate:\", str(final[\"rate\"]) + \"/s\")\n", - " print(\"Errors:\", final[\"errors\"])\n", - " return final" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Create Benchmark Argo Workflow\n", - "\n", - "In order to create a benchmark, we created a simple argo workflow template so you can leverage the power of the helm charts.\n", - "\n", - "Before we dive into the contents of the full helm chart, let's first give it a try with some of the settings.\n", - "\n", - "We will run a batch job that will set up a Seldon Deployment with 1 replicas and 4 cpus (with 100 max workers) to send requests." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "scrolled": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name: seldon-benchmark-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: default\r\n", - "Status: Pending\r\n", - "Created: Fri Aug 07 18:09:40 +0100 (now)\r\n" - ] - } - ], - "source": [ - "!helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \\\n", - " --set workflow.name=seldon-benchmark-process \\\n", - " --set seldonDeployment.name=sklearn \\\n", - " --set seldonDeployment.replicas=1 \\\n", - " --set seldonDeployment.serverWorkers=1 \\\n", - " --set seldonDeployment.serverThreads=10 \\\n", - " --set seldonDeployment.modelUri=\"gs://seldon-models/sklearn/iris\" \\\n", - " --set seldonDeployment.server=\"SKLEARN_SERVER\" \\\n", - " --set seldonDeployment.apiType=rest \\\n", - " --set benchmark.cpus=4 \\\n", - " --set benchmark.maxWorkers=100 \\\n", - " --set benchmark.duration=30s \\\n", - " --set benchmark.rate=0 \\\n", - " --set benchmark.data='\\{\"data\": {\"ndarray\": [[0\\,1\\,2\\,3]]\\}\\}' \\\n", - " | argo submit -" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME STATUS AGE DURATION PRIORITY\r\n", - "seldon-benchmark-process Succeeded 2m 1m 0\r\n" - ] - } - ], - "source": [ - "!argo list" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name: seldon-benchmark-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: default\r\n", - "Status: Succeeded\r\n", - "Created: Fri Aug 07 18:09:40 +0100 (2 minutes ago)\r\n", - "Started: Fri Aug 07 18:09:40 +0100 (2 minutes ago)\r\n", - "Finished: Fri Aug 07 18:11:09 +0100 (51 seconds ago)\r\n", - "Duration: 1 minute 29 seconds\r\n", - "\r\n", - "\u001b[39mSTEP\u001b[0m PODNAME DURATION MESSAGE\r\n", - " \u001b[32m✔\u001b[0m seldon-benchmark-process (seldon-benchmark-process) \r\n", - " ├---\u001b[32m✔\u001b[0m create-seldon-resource (create-seldon-resource-template) seldon-benchmark-process-3980407503 2s \r\n", - " ├---\u001b[32m✔\u001b[0m wait-seldon-resource (wait-seldon-resource-template) seldon-benchmark-process-2136965893 49s \r\n", - " └---\u001b[32m✔\u001b[0m run-benchmark (run-benchmark-template) seldon-benchmark-process-780051119 32s \r\n" - ] - } - ], - "source": [ - "!argo get seldon-benchmark-process" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:41.804Z\" level=info msg=\"Starting Workflow Executor\" version=v2.9.3\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:41.809Z\" level=info msg=\"Creating a docker executor\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:41.809Z\" level=info msg=\"Executor (version: v2.9.3, build_date: 2020-07-18T19:11:19Z) initialized (pod: default/seldon-benchmark-process-3980407503) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-benchmark-process\\\\\\\"\\\\n uid: \\\\\\\"e0364966-b2c1-4ee7-a7cf-421952ba3c7a\\\\\\\"\\\\nspec:\\\\n annotations:\\\\n seldon.io/executor: \\\\\\\"false\\\\\\\"\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n transport: rest\\\\n predictors:\\\\n - componentSpecs:\\\\n - spec:\\\\n containers:\\\\n - name: classifier\\\\n env:\\\\n - name: GUNICORN_THREADS\\\\n value: 10\\\\n - name: GUNICORN_WORKERS\\\\n value: 1\\\\n resources:\\\\n requests:\\\\n cpu: 50m\\\\n memory: 100Mi\\\\n limits:\\\\n cpu: 50m\\\\n memory: 1000Mi\\\\n graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 1\\\\n\\\"}}\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:41.809Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:41.810Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:42.456Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:09:42.457Z\" level=info msg=\"No output parameters\"\r\n", - "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\r\n", - "\u001b[32mwait-seldon-resource\u001b[0m:\tdeployment \"sklearn-default-0-classifier\" successfully rolled out\r\n", - "\u001b[35mrun-benchmark\u001b[0m:\t{\"latencies\":{\"total\":3011298973622,\"mean\":339033885,\"50th\":272840630,\"90th\":339539236,\"95th\":368299307,\"99th\":4982426813,\"max\":5597505277,\"min\":206244298},\"bytes_in\":{\"total\":3081764,\"mean\":346.9673496960144},\"bytes_out\":{\"total\":301988,\"mean\":34},\"earliest\":\"2020-08-07T17:10:37.117884325Z\",\"latest\":\"2020-08-07T17:11:07.118729145Z\",\"end\":\"2020-08-07T17:11:07.366654843Z\",\"duration\":30000844820,\"wait\":247925698,\"requests\":8882,\"rate\":296.05832946673667,\"throughput\":293.63176909007353,\"success\":1,\"status_codes\":{\"200\":8882},\"errors\":[]}\r\n" - ] - } - ], - "source": [ - "!argo logs -w seldon-benchmark-process" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Latencies:\n", - "\tmean: 339.033885 ms\n", - "\t50th: 272.84063 ms\n", - "\t90th: 339.539236 ms\n", - "\t95th: 368.299307 ms\n", - "\t99th: 4982.426813 ms\n", - "\n", - "Throughput: 293.63176909007353/s\n", - "Errors: False\n" - ] - } - ], - "source": [ - "import json\n", - "wf_logs = !argo logs -w seldon-benchmark-process \n", - "wf_bench = wf_logs[-1]\n", - "wf_json_str = wf_bench[24:]\n", - "results = json.loads(wf_json_str)\n", - "\n", - "print(\"Latencies:\")\n", - "print(\"\\tmean:\", results[\"latencies\"][\"mean\"] / 1e6, \"ms\")\n", - "print(\"\\t50th:\", results[\"latencies\"][\"50th\"] / 1e6, \"ms\")\n", - "print(\"\\t90th:\", results[\"latencies\"][\"90th\"] / 1e6, \"ms\")\n", - "print(\"\\t95th:\", results[\"latencies\"][\"95th\"] / 1e6, \"ms\")\n", - "print(\"\\t99th:\", results[\"latencies\"][\"99th\"] / 1e6, \"ms\")\n", - "print(\"\")\n", - "print(\"Throughput:\", str(results[\"throughput\"]) + \"/s\")\n", - "print(\"Errors:\", len(results[\"errors\"]) > 0)" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workflow 'seldon-benchmark-process' deleted\r\n" - ] - } - ], - "source": [ - "!argo delete seldon-benchmark-process" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Create GRPC benchmark with GHZ and Argo Workflows " - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name: seldon-benchmark-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: default\r\n", - "Status: Pending\r\n", - "Created: Fri Aug 07 18:22:38 +0100 (now)\r\n" - ] - } - ], - "source": [ - "!helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \\\n", - " --set workflow.name=seldon-benchmark-process \\\n", - " --set seldonDeployment.name=sklearn \\\n", - " --set seldonDeployment.replicas=1 \\\n", - " --set seldonDeployment.serverWorkers=1 \\\n", - " --set seldonDeployment.serverThreads=10 \\\n", - " --set seldonDeployment.modelUri=\"gs://seldon-models/sklearn/iris\" \\\n", - " --set seldonDeployment.server=\"SKLEARN_SERVER\" \\\n", - " --set seldonDeployment.apiType=grpc \\\n", - " --set benchmark.cpus=4 \\\n", - " --set benchmark.maxWorkers=100 \\\n", - " --set benchmark.duration=\"120s\" \\\n", - " --set benchmark.rate=0 \\\n", - " --set benchmark.data='\\{\"data\": {\"ndarray\": [[0\\,1\\,2\\,3]]\\}\\}' \\\n", - " | argo submit -" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "NAME STATUS AGE DURATION PRIORITY\r\n", - "seldon-benchmark-process Succeeded 4m 2m 0\r\n" - ] - } - ], - "source": [ - "!argo list" - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name: seldon-benchmark-process\r\n", - "Namespace: default\r\n", - "ServiceAccount: default\r\n", - "Status: Succeeded\r\n", - "Created: Fri Aug 07 18:22:38 +0100 (4 minutes ago)\r\n", - "Started: Fri Aug 07 18:22:38 +0100 (4 minutes ago)\r\n", - "Finished: Fri Aug 07 18:25:11 +0100 (1 minute ago)\r\n", - "Duration: 2 minutes 33 seconds\r\n", - "\r\n", - "\u001b[39mSTEP\u001b[0m PODNAME DURATION MESSAGE\r\n", - " \u001b[32m✔\u001b[0m seldon-benchmark-process (seldon-benchmark-process) \r\n", - " ├---\u001b[32m✔\u001b[0m create-seldon-resource (create-seldon-resource-template) seldon-benchmark-process-3980407503 2s \r\n", - " ├---\u001b[32m✔\u001b[0m wait-seldon-resource (wait-seldon-resource-template) seldon-benchmark-process-2136965893 26s \r\n", - " └---\u001b[32m✔\u001b[0m run-benchmark (run-benchmark-template) seldon-benchmark-process-780051119 2m \r\n" - ] - } - ], - "source": [ - "!argo get seldon-benchmark-process" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:39.446Z\" level=info msg=\"Starting Workflow Executor\" version=v2.9.3\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:39.450Z\" level=info msg=\"Creating a docker executor\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:39.450Z\" level=info msg=\"Executor (version: v2.9.3, build_date: 2020-07-18T19:11:19Z) initialized (pod: default/seldon-benchmark-process-3980407503) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"arguments\\\":{},\\\"inputs\\\":{},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n namespace: default\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-benchmark-process\\\\\\\"\\\\n uid: \\\\\\\"e472d69d-44ed-4a45-86b3-d4b64146002b\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn\\\\\\\"\\\\n transport: grpc\\\\n predictors:\\\\n - componentSpecs:\\\\n - spec:\\\\n containers:\\\\n - name: classifier\\\\n env:\\\\n - name: GUNICORN_THREADS\\\\n value: 10\\\\n - name: GUNICORN_WORKERS\\\\n value: 1\\\\n graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 1\\\\n\\\"}}\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:39.450Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:39.450Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:40.060Z\" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn\r\n", - "\u001b[35mcreate-seldon-resource\u001b[0m:\ttime=\"2020-08-07T17:22:40.060Z\" level=info msg=\"No output parameters\"\r\n", - "\u001b[32mwait-seldon-resource\u001b[0m:\tWaiting for deployment \"sklearn-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\r\n", - "\u001b[32mwait-seldon-resource\u001b[0m:\tdeployment \"sklearn-default-0-classifier\" successfully rolled out\r\n", - "\u001b[35mrun-benchmark\u001b[0m:\t{\"date\":\"2020-08-07T17:25:09Z\",\"endReason\":\"timeout\",\"options\":{\"host\":\"istio-ingressgateway.istio-system.svc.cluster.local:80\",\"proto\":\"/proto/prediction.proto\",\"import-paths\":[\"/proto\",\".\"],\"call\":\"seldon.protos.Seldon/Predict\",\"insecure\":true,\"total\":2147483647,\"concurrency\":50,\"connections\":1,\"duration\":120000000000,\"timeout\":20000000000,\"dial-timeout\":10000000000,\"data\":{\"data\":{\"ndarray\":[[0,1,2,3]]}},\"binary\":false,\"metadata\":{\"namespace\":\"default\",\"seldon\":\"sklearn\"},\"CPUs\":4},\"count\":88874,\"total\":120001033613,\"average\":67376309,\"fastest\":21863600,\"slowest\":148816057,\"rps\":740.6102874631579,\"errorDistribution\":{\"rpc error: code = Unavailable desc = transport is closing\":50},\"statusCodeDistribution\":{\"OK\":88824,\"Unavailable\":50},\"latencyDistribution\":[{\"percentage\":10,\"latency\":54583101},{\"percentage\":25,\"latency\":59326600},{\"percentage\":50,\"latency\":65257398},{\"percentage\":75,\"latency\":73167799},{\"percentage\":90,\"latency\":82939600},{\"percentage\":95,\"latency\":89598800},{\"percentage\":99,\"latency\":101463001}]}\r\n" - ] - } - ], - "source": [ - "!argo logs -w seldon-benchmark-process" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Latencies:\n", - "\tmean: 67.376309 ms\n", - "\t50th: 65.257398 ms\n", - "\t90th: 82.9396 ms\n", - "\t95th: 89.5988 ms\n", - "\t99th: 101.463001 ms\n", - "\n", - "Rate: 740.6102874631579/s\n", - "Errors: True\n", - "Errors: {'OK': 88824, 'Unavailable': 50}\n" - ] - } - ], - "source": [ - "import json\n", - "wf_logs = !argo logs -w seldon-benchmark-process \n", - "wf_bench = wf_logs[-1]\n", - "wf_json_str = wf_bench[24:]\n", - "results = json.loads(wf_json_str)\n", - "\n", - "print(\"Latencies:\")\n", - "print(\"\\tmean:\", results[\"average\"] / 1e6, \"ms\")\n", - "print(\"\\t50th:\", results[\"latencyDistribution\"][-5][\"latency\"] / 1e6, \"ms\")\n", - "print(\"\\t90th:\", results[\"latencyDistribution\"][-3][\"latency\"] / 1e6, \"ms\")\n", - "print(\"\\t95th:\", results[\"latencyDistribution\"][-2][\"latency\"] / 1e6, \"ms\")\n", - "print(\"\\t99th:\", results[\"latencyDistribution\"][-1][\"latency\"] / 1e6, \"ms\")\n", - "print(\"\")\n", - "print(\"Rate:\", str(results[\"rps\"]) + \"/s\")\n", - "print(\"Errors:\", results[\"statusCodeDistribution\"].get(\"Unavailable\", 0) > 0)\n", - "print(\"Errors:\", results[\"statusCodeDistribution\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Workflow 'seldon-benchmark-process' deleted\r\n" - ] - } - ], - "source": [ - "!argo delete seldon-benchmark-process" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run a set of tests\n", - "\n", - "We can now leverage the helm charts we created above to run a grid search on a set of parameters." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import itertools as it\n", - "import json\n", - "import time\n", - "\n", - "grid_opts = {\n", - " \"A-replicas\": [1, 3],\n", - " \"B-serverWorkers\": [1, 4],\n", - " \"C-serverThreads\": [50, 200],\n", - " \"D-apiType\": [\"rest\", \"grpc\"],\n", - " \"E-cpus\": [1, 4],\n", - " \"F-maxWorkers\": [100, 300],\n", - " \"G-useEngine\": [\"true\", \"false\"],\n", - "}\n", - "\n", - "allNames = sorted(grid_opts)\n", - "combinations = it.product(*(grid_opts[Name] for Name in allNames))\n", - "all_results = []\n", - "for curr_values in combinations:\n", - " print(\"VALUES:\", curr_values)\n", - " replicas, server_workers, server_threads, api_type, cpus, max_wokers, use_engine = curr_values\n", - "\n", - " # For some reason python vars don't work with multiline helm charts\n", - " %env REPLICAS=$replicas\n", - " %env SERVER_WORKERS=$server_workers\n", - " %env SERVER_THREADS=$server_threads\n", - " %env API_TYPE=$api_type\n", - " %env CPUS=$cpus\n", - " %env MAX_WORKERS=$max_wokers\n", - " %env USE_ENGINE=$use_engine\n", - " \n", - " !helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \\\n", - " --set workflow.name=seldon-benchmark-process \\\n", - " --set seldonDeployment.name=sklearn \\\n", - " --set seldonDeployment.replicas=$REPLICAS \\\n", - " --set seldonDeployment.serverWorkers=$SERVER_WORKERS \\\n", - " --set seldonDeployment.serverThreads=$SERVER_THREADS \\\n", - " --set seldonDeployment.apiType=$API_TYPE \\\n", - " --set seldonDeployment.useEngine=\\\"$USE_ENGINE\\\" \\\n", - " --set benchmark.cpus=$CPUS \\\n", - " --set benchmark.maxWorkers=$MAX_WORKERS \\\n", - " --set benchmark.duration=120s \\\n", - " --set benchmark.rate=0 \\\n", - " --set benchmark.data='\\{\"data\": {\"ndarray\": [[0\\,1\\,2\\,3]]\\}\\}' \\\n", - " | argo submit --wait -\n", - " \n", - " !argo wait seldon-benchmark-process \n", - " \n", - " wf_logs = !argo logs -w seldon-benchmark-process \n", - " wf_bench = wf_logs[-1]\n", - " wf_json_str = wf_bench[24:]\n", - " results = json.loads(wf_json_str)\n", - " \n", - " result = get_results(results)\n", - " result[\"replicas\"] = replicas\n", - " result[\"server_workers\"] = server_workers\n", - " result[\"server_threads\"] = server_threads\n", - " result[\"apiType\"] = api_type\n", - " result[\"cpus\"] = cpus\n", - " result[\"max_wokers\"] = max_wokers\n", - " result[\"use_engine\"] = use_engine\n", - " all_results.append(result)\n", - " \n", - " !argo delete seldon-benchmark-process\n", - " time.sleep(1)\n", - " print(\"\\n\\n\")\n", - " " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Deeper Analysis\n", - "Now that we have all the parameters, we can do a deeper analysis" - ] - }, - { - "cell_type": "code", - "execution_count": 186, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
01150rest1200true489.269344455.617128612.294382672.510108832.322767407.879172[]
11150rest1200false529.767457514.151876591.278115621.463805749.348556376.649458[]
21150rest4200true547.618426526.472215661.947413720.039676863.596098364.363839[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
41150grpc1200true95.32294397.896699117.221999125.852400141.615501523.628160{'OK': 62790, 'Unavailable': 50}
\n", - "
" - ], - "text/plain": [ - " replicas server_workers server_threads apiType cpus max_wokers use_engine \\\n", - "0 1 1 50 rest 1 200 true \n", - "1 1 1 50 rest 1 200 false \n", - "2 1 1 50 rest 4 200 true \n", - "3 1 1 50 rest 4 200 false \n", - "4 1 1 50 grpc 1 200 true \n", - "\n", - " mean 50th 90th 95th 99th rate \\\n", - "0 489.269344 455.617128 612.294382 672.510108 832.322767 407.879172 \n", - "1 529.767457 514.151876 591.278115 621.463805 749.348556 376.649458 \n", - "2 547.618426 526.472215 661.947413 720.039676 863.596098 364.363839 \n", - "3 593.880113 602.945695 737.993290 770.777543 1003.510371 336.075411 \n", - "4 95.322943 97.896699 117.221999 125.852400 141.615501 523.628160 \n", - "\n", - " errors \n", - "0 [] \n", - "1 [] \n", - "2 [] \n", - "3 [] \n", - "4 {'OK': 62790, 'Unavailable': 50} " - ] - }, - "execution_count": 186, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import pandas as pd\n", - "\n", - "df = pd.DataFrame.from_dict(results)\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### GRPC as expected outperforms REST" - ] - }, - { - "cell_type": "code", - "execution_count": 189, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
6034200grpc1200true31.38986123.76958971.58379578.88139891.3127971586.593680{'OK': 190361, 'Unavailable': 48}
523450grpc1200true31.39845126.31300064.84151573.03580088.7441981586.555365{'OK': 190333, 'Unavailable': 71}
4531200grpc1200false32.19124030.44830260.61630168.72440691.4843081547.003054{'OK': 185606, 'Unavailable': 49}
6134200grpc1200false32.72767428.48340063.75079672.59731090.6938121521.590875{'OK': 182555, 'Unavailable': 49}
553450grpc4200false33.62984829.61070167.06589577.77310097.2965991479.320474{'OK': 177471, 'Unavailable': 50}
.............................................
1011200rest4200true571.452398556.699256693.093315751.1975981024.233714348.889260[]
1111200rest4200false587.900216556.869872723.744376774.244702939.994423339.396160[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
811200rest1200true633.043624617.853285741.229073776.5605781846.623159314.908167[]
911200rest1200false641.530606653.922529802.558303847.4144841570.484029310.839312[]
\n", - "

64 rows × 14 columns

\n", - "
" - ], - "text/plain": [ - " replicas server_workers server_threads apiType cpus max_wokers use_engine \\\n", - "60 3 4 200 grpc 1 200 true \n", - "52 3 4 50 grpc 1 200 true \n", - "45 3 1 200 grpc 1 200 false \n", - "61 3 4 200 grpc 1 200 false \n", - "55 3 4 50 grpc 4 200 false \n", - ".. ... ... ... ... ... ... ... \n", - "10 1 1 200 rest 4 200 true \n", - "11 1 1 200 rest 4 200 false \n", - "3 1 1 50 rest 4 200 false \n", - "8 1 1 200 rest 1 200 true \n", - "9 1 1 200 rest 1 200 false \n", - "\n", - " mean 50th 90th 95th 99th rate \\\n", - "60 31.389861 23.769589 71.583795 78.881398 91.312797 1586.593680 \n", - "52 31.398451 26.313000 64.841515 73.035800 88.744198 1586.555365 \n", - "45 32.191240 30.448302 60.616301 68.724406 91.484308 1547.003054 \n", - "61 32.727674 28.483400 63.750796 72.597310 90.693812 1521.590875 \n", - "55 33.629848 29.610701 67.065895 77.773100 97.296599 1479.320474 \n", - ".. ... ... ... ... ... ... \n", - "10 571.452398 556.699256 693.093315 751.197598 1024.233714 348.889260 \n", - "11 587.900216 556.869872 723.744376 774.244702 939.994423 339.396160 \n", - "3 593.880113 602.945695 737.993290 770.777543 1003.510371 336.075411 \n", - "8 633.043624 617.853285 741.229073 776.560578 1846.623159 314.908167 \n", - "9 641.530606 653.922529 802.558303 847.414484 1570.484029 310.839312 \n", - "\n", - " errors \n", - "60 {'OK': 190361, 'Unavailable': 48} \n", - "52 {'OK': 190333, 'Unavailable': 71} \n", - "45 {'OK': 185606, 'Unavailable': 49} \n", - "61 {'OK': 182555, 'Unavailable': 49} \n", - "55 {'OK': 177471, 'Unavailable': 50} \n", - ".. ... \n", - "10 [] \n", - "11 [] \n", - "3 [] \n", - "8 [] \n", - "9 [] \n", - "\n", - "[64 rows x 14 columns]" - ] - }, - "execution_count": 189, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df.sort_values(\"rate\", ascending=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deeper dive REST\n", - "As expected replicas has the biggest impact. It seems the parameters on the benchmark worker don't seem to affect throughput." - ] - }, - { - "cell_type": "code", - "execution_count": 190, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
4131200rest1200false201.1675468.844305629.250888690.807158809.635194992.298652[]
483450rest1200true208.42957611.377699655.466848685.265506758.664504957.846772[]
513450rest4200false211.22852613.592301641.484819675.713639795.682869945.090980[]
5934200rest4200false214.35883413.573121670.449768690.048496722.537613930.694079[]
5734200rest1200false216.6463209.336961684.733598704.485018733.636276921.350903[]
4031200rest1200true217.72239716.593757657.144743695.158232745.726065916.803160[]
323150rest1200true218.81795210.808913689.809571757.737985867.650689912.589694[]
5634200rest1200true221.0318769.197338690.217169711.800471742.657817903.072311[]
503450rest4200true221.26324916.583482688.637696711.870214781.197685902.315850[]
5834200rest4200true221.56695611.037262685.417461713.923684771.814053901.132352[]
353150rest4200false225.71911415.998348704.701196741.890962852.664830884.187996[]
333150rest1200false229.6533669.844413725.066803775.186525857.762245869.461119[]
4231200rest4200true231.01653615.829218737.382688788.027859885.482116863.960992[]
493450rest1200false231.98692711.193407702.083677769.889421901.360146860.495277[]
4331200rest4200false239.15079414.147647722.982655789.211063929.436195834.381347[]
343150rest4200true240.088790121.078205707.862815771.405571965.932529831.402721[]
2614200rest4200true413.608259409.729690442.576049460.804621502.762769482.699096[]
171450rest1200false429.042835412.423403500.170846522.423418586.685379465.431891[]
2714200rest4200false432.609142426.606234488.443435512.393140556.238288461.578501[]
2514200rest1200false463.422714450.181537551.644801602.270942670.647806430.891782[]
161450rest1200true475.510231456.056479583.716159650.365364746.791628419.975983[]
191450rest4200false481.143061450.734477602.026223689.302618863.072782414.795159[]
181450rest4200true488.185779436.842244628.922397735.5126541068.474298408.992844[]
01150rest1200true489.269344455.617128612.294382672.510108832.322767407.879172[]
2414200rest1200true514.472545488.358257591.629431631.3928131517.062374387.882855[]
11150rest1200false529.767457514.151876591.278115621.463805749.348556376.649458[]
21150rest4200true547.618426526.472215661.947413720.039676863.596098364.363839[]
1011200rest4200true571.452398556.699256693.093315751.1975981024.233714348.889260[]
1111200rest4200false587.900216556.869872723.744376774.244702939.994423339.396160[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
811200rest1200true633.043624617.853285741.229073776.5605781846.623159314.908167[]
911200rest1200false641.530606653.922529802.558303847.4144841570.484029310.839312[]
\n", - "
" - ], - "text/plain": [ - " replicas server_workers server_threads apiType cpus max_wokers use_engine \\\n", - "41 3 1 200 rest 1 200 false \n", - "48 3 4 50 rest 1 200 true \n", - "51 3 4 50 rest 4 200 false \n", - "59 3 4 200 rest 4 200 false \n", - "57 3 4 200 rest 1 200 false \n", - "40 3 1 200 rest 1 200 true \n", - "32 3 1 50 rest 1 200 true \n", - "56 3 4 200 rest 1 200 true \n", - "50 3 4 50 rest 4 200 true \n", - "58 3 4 200 rest 4 200 true \n", - "35 3 1 50 rest 4 200 false \n", - "33 3 1 50 rest 1 200 false \n", - "42 3 1 200 rest 4 200 true \n", - "49 3 4 50 rest 1 200 false \n", - "43 3 1 200 rest 4 200 false \n", - "34 3 1 50 rest 4 200 true \n", - "26 1 4 200 rest 4 200 true \n", - "17 1 4 50 rest 1 200 false \n", - "27 1 4 200 rest 4 200 false \n", - "25 1 4 200 rest 1 200 false \n", - "16 1 4 50 rest 1 200 true \n", - "19 1 4 50 rest 4 200 false \n", - "18 1 4 50 rest 4 200 true \n", - "0 1 1 50 rest 1 200 true \n", - "24 1 4 200 rest 1 200 true \n", - "1 1 1 50 rest 1 200 false \n", - "2 1 1 50 rest 4 200 true \n", - "10 1 1 200 rest 4 200 true \n", - "11 1 1 200 rest 4 200 false \n", - "3 1 1 50 rest 4 200 false \n", - "8 1 1 200 rest 1 200 true \n", - "9 1 1 200 rest 1 200 false \n", - "\n", - " mean 50th 90th 95th 99th rate \\\n", - "41 201.167546 8.844305 629.250888 690.807158 809.635194 992.298652 \n", - "48 208.429576 11.377699 655.466848 685.265506 758.664504 957.846772 \n", - "51 211.228526 13.592301 641.484819 675.713639 795.682869 945.090980 \n", - "59 214.358834 13.573121 670.449768 690.048496 722.537613 930.694079 \n", - "57 216.646320 9.336961 684.733598 704.485018 733.636276 921.350903 \n", - "40 217.722397 16.593757 657.144743 695.158232 745.726065 916.803160 \n", - "32 218.817952 10.808913 689.809571 757.737985 867.650689 912.589694 \n", - "56 221.031876 9.197338 690.217169 711.800471 742.657817 903.072311 \n", - "50 221.263249 16.583482 688.637696 711.870214 781.197685 902.315850 \n", - "58 221.566956 11.037262 685.417461 713.923684 771.814053 901.132352 \n", - "35 225.719114 15.998348 704.701196 741.890962 852.664830 884.187996 \n", - "33 229.653366 9.844413 725.066803 775.186525 857.762245 869.461119 \n", - "42 231.016536 15.829218 737.382688 788.027859 885.482116 863.960992 \n", - "49 231.986927 11.193407 702.083677 769.889421 901.360146 860.495277 \n", - "43 239.150794 14.147647 722.982655 789.211063 929.436195 834.381347 \n", - "34 240.088790 121.078205 707.862815 771.405571 965.932529 831.402721 \n", - "26 413.608259 409.729690 442.576049 460.804621 502.762769 482.699096 \n", - "17 429.042835 412.423403 500.170846 522.423418 586.685379 465.431891 \n", - "27 432.609142 426.606234 488.443435 512.393140 556.238288 461.578501 \n", - "25 463.422714 450.181537 551.644801 602.270942 670.647806 430.891782 \n", - "16 475.510231 456.056479 583.716159 650.365364 746.791628 419.975983 \n", - "19 481.143061 450.734477 602.026223 689.302618 863.072782 414.795159 \n", - "18 488.185779 436.842244 628.922397 735.512654 1068.474298 408.992844 \n", - "0 489.269344 455.617128 612.294382 672.510108 832.322767 407.879172 \n", - "24 514.472545 488.358257 591.629431 631.392813 1517.062374 387.882855 \n", - "1 529.767457 514.151876 591.278115 621.463805 749.348556 376.649458 \n", - "2 547.618426 526.472215 661.947413 720.039676 863.596098 364.363839 \n", - "10 571.452398 556.699256 693.093315 751.197598 1024.233714 348.889260 \n", - "11 587.900216 556.869872 723.744376 774.244702 939.994423 339.396160 \n", - "3 593.880113 602.945695 737.993290 770.777543 1003.510371 336.075411 \n", - "8 633.043624 617.853285 741.229073 776.560578 1846.623159 314.908167 \n", - "9 641.530606 653.922529 802.558303 847.414484 1570.484029 310.839312 \n", - "\n", - " errors \n", - "41 [] \n", - "48 [] \n", - "51 [] \n", - "59 [] \n", - "57 [] \n", - "40 [] \n", - "32 [] \n", - "56 [] \n", - "50 [] \n", - "58 [] \n", - "35 [] \n", - "33 [] \n", - "42 [] \n", - "49 [] \n", - "43 [] \n", - "34 [] \n", - "26 [] \n", - "17 [] \n", - "27 [] \n", - "25 [] \n", - "16 [] \n", - "19 [] \n", - "18 [] \n", - "0 [] \n", - "24 [] \n", - "1 [] \n", - "2 [] \n", - "10 [] \n", - "11 [] \n", - "3 [] \n", - "8 [] \n", - "9 [] " - ] - }, - "execution_count": 190, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[df[\"apiType\"] == \"rest\"].sort_values(\"rate\", ascending=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### Deep dive on GRPC" - ] - }, - { - "cell_type": "code", - "execution_count": 191, - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
6034200grpc1200true31.38986123.76958971.58379578.88139891.3127971586.593680{'OK': 190361, 'Unavailable': 48}
523450grpc1200true31.39845126.31300064.84151573.03580088.7441981586.555365{'OK': 190333, 'Unavailable': 71}
4531200grpc1200false32.19124030.44830260.61630168.72440691.4843081547.003054{'OK': 185606, 'Unavailable': 49}
6134200grpc1200false32.72767428.48340063.75079672.59731090.6938121521.590875{'OK': 182555, 'Unavailable': 49}
553450grpc4200false33.62984829.61070167.06589577.77310097.2965991479.320474{'OK': 177471, 'Unavailable': 50}
4731200grpc4200false33.86102330.20740070.27269883.485103105.6393011469.503585{'OK': 176302, 'Unavailable': 50}
6234200grpc4200true34.74680131.89658572.73279684.03276399.4330901432.045405{'OK': 171799, 'Unavailable': 50}
543450grpc4200true34.78688332.14119772.55431382.64970295.0497051430.209225{'OK': 171578, 'Unavailable': 49}
373150grpc1200false35.14937635.15318762.84280072.79180094.2402991416.745392{'OK': 169973, 'Unavailable': 50}
363150grpc1200true35.16765731.85930065.64489576.24079998.9258991415.967279{'OK': 169880, 'Unavailable': 48}
4631200grpc4200true35.28617324.98850083.07930194.264796111.4488951410.595798{'OK': 169202, 'Unavailable': 71}
533450grpc1200false35.54394030.52890069.44989582.465882100.3811951400.945365{'OK': 168074, 'Unavailable': 50}
6334200grpc4200false35.70618130.17530076.12170185.84238599.0727011393.469861{'OK': 167180, 'Unavailable': 49}
393150grpc4200false36.02680433.54119269.94279881.321704108.5289011381.482907{'OK': 165711, 'Unavailable': 69}
383150grpc4200true36.32571835.59849873.21199782.948302102.2483971369.739820{'OK': 164333, 'Unavailable': 49}
4431200grpc1200true37.32656135.60938870.52259879.731401101.2974001334.058278{'OK': 160053, 'Unavailable': 50}
2914200grpc1200false63.24012961.51920172.90500077.14070089.520499789.347786{'OK': 94678, 'Unavailable': 50}
2814200grpc1200true63.53711961.85520074.29910079.87660197.179900785.631011{'OK': 94233, 'Unavailable': 50}
201450grpc1200true65.71157764.22050078.08530083.56360094.907700759.690398{'OK': 91119, 'Unavailable': 50}
211450grpc1200false66.89814363.42080083.83710092.332400108.138499746.209307{'OK': 89501, 'Unavailable': 50}
3014200grpc4200true67.21160965.50420079.98989986.808200106.460500742.433252{'OK': 89044, 'Unavailable': 50}
71150grpc4200false67.77063262.16850488.674303102.537000120.848185736.385539{'OK': 88318, 'Unavailable': 49}
3114200grpc4200false70.57783468.97289984.86920089.875600102.761897707.046156{'OK': 84796, 'Unavailable': 50}
221450grpc4200true70.81841167.59160087.91410497.004000115.388900704.647865{'OK': 84514, 'Unavailable': 50}
1511200grpc4200false71.57162769.34870091.60959898.471998111.237797697.252435{'OK': 83622, 'Unavailable': 50}
231450grpc4200false73.85378070.60470191.03140098.064600116.658902675.704389{'OK': 81035, 'Unavailable': 50}
1411200grpc4200true89.66250087.678702107.762199118.226099146.838610556.478774{'OK': 66728, 'Unavailable': 50}
61150grpc4200true90.65502591.964500108.453597116.581800148.048199550.406903{'OK': 66003, 'Unavailable': 50}
51150grpc1200false92.93040093.020601113.056104122.476104150.119004537.076992{'OK': 64405, 'Unavailable': 50}
1211200grpc1200true94.69595194.988002111.319799118.210000134.270997527.054914{'OK': 63202, 'Unavailable': 50}
41150grpc1200true95.32294397.896699117.221999125.852400141.615501523.628160{'OK': 62790, 'Unavailable': 50}
1311200grpc1200false96.01629697.410200113.779899120.184499136.929395519.810588{'OK': 62332, 'Unavailable': 50}
\n", - "
" - ], - "text/plain": [ - " replicas server_workers server_threads apiType cpus max_wokers use_engine \\\n", - "60 3 4 200 grpc 1 200 true \n", - "52 3 4 50 grpc 1 200 true \n", - "45 3 1 200 grpc 1 200 false \n", - "61 3 4 200 grpc 1 200 false \n", - "55 3 4 50 grpc 4 200 false \n", - "47 3 1 200 grpc 4 200 false \n", - "62 3 4 200 grpc 4 200 true \n", - "54 3 4 50 grpc 4 200 true \n", - "37 3 1 50 grpc 1 200 false \n", - "36 3 1 50 grpc 1 200 true \n", - "46 3 1 200 grpc 4 200 true \n", - "53 3 4 50 grpc 1 200 false \n", - "63 3 4 200 grpc 4 200 false \n", - "39 3 1 50 grpc 4 200 false \n", - "38 3 1 50 grpc 4 200 true \n", - "44 3 1 200 grpc 1 200 true \n", - "29 1 4 200 grpc 1 200 false \n", - "28 1 4 200 grpc 1 200 true \n", - "20 1 4 50 grpc 1 200 true \n", - "21 1 4 50 grpc 1 200 false \n", - "30 1 4 200 grpc 4 200 true \n", - "7 1 1 50 grpc 4 200 false \n", - "31 1 4 200 grpc 4 200 false \n", - "22 1 4 50 grpc 4 200 true \n", - "15 1 1 200 grpc 4 200 false \n", - "23 1 4 50 grpc 4 200 false \n", - "14 1 1 200 grpc 4 200 true \n", - "6 1 1 50 grpc 4 200 true \n", - "5 1 1 50 grpc 1 200 false \n", - "12 1 1 200 grpc 1 200 true \n", - "4 1 1 50 grpc 1 200 true \n", - "13 1 1 200 grpc 1 200 false \n", - "\n", - " mean 50th 90th 95th 99th rate \\\n", - "60 31.389861 23.769589 71.583795 78.881398 91.312797 1586.593680 \n", - "52 31.398451 26.313000 64.841515 73.035800 88.744198 1586.555365 \n", - "45 32.191240 30.448302 60.616301 68.724406 91.484308 1547.003054 \n", - "61 32.727674 28.483400 63.750796 72.597310 90.693812 1521.590875 \n", - "55 33.629848 29.610701 67.065895 77.773100 97.296599 1479.320474 \n", - "47 33.861023 30.207400 70.272698 83.485103 105.639301 1469.503585 \n", - "62 34.746801 31.896585 72.732796 84.032763 99.433090 1432.045405 \n", - "54 34.786883 32.141197 72.554313 82.649702 95.049705 1430.209225 \n", - "37 35.149376 35.153187 62.842800 72.791800 94.240299 1416.745392 \n", - "36 35.167657 31.859300 65.644895 76.240799 98.925899 1415.967279 \n", - "46 35.286173 24.988500 83.079301 94.264796 111.448895 1410.595798 \n", - "53 35.543940 30.528900 69.449895 82.465882 100.381195 1400.945365 \n", - "63 35.706181 30.175300 76.121701 85.842385 99.072701 1393.469861 \n", - "39 36.026804 33.541192 69.942798 81.321704 108.528901 1381.482907 \n", - "38 36.325718 35.598498 73.211997 82.948302 102.248397 1369.739820 \n", - "44 37.326561 35.609388 70.522598 79.731401 101.297400 1334.058278 \n", - "29 63.240129 61.519201 72.905000 77.140700 89.520499 789.347786 \n", - "28 63.537119 61.855200 74.299100 79.876601 97.179900 785.631011 \n", - "20 65.711577 64.220500 78.085300 83.563600 94.907700 759.690398 \n", - "21 66.898143 63.420800 83.837100 92.332400 108.138499 746.209307 \n", - "30 67.211609 65.504200 79.989899 86.808200 106.460500 742.433252 \n", - "7 67.770632 62.168504 88.674303 102.537000 120.848185 736.385539 \n", - "31 70.577834 68.972899 84.869200 89.875600 102.761897 707.046156 \n", - "22 70.818411 67.591600 87.914104 97.004000 115.388900 704.647865 \n", - "15 71.571627 69.348700 91.609598 98.471998 111.237797 697.252435 \n", - "23 73.853780 70.604701 91.031400 98.064600 116.658902 675.704389 \n", - "14 89.662500 87.678702 107.762199 118.226099 146.838610 556.478774 \n", - "6 90.655025 91.964500 108.453597 116.581800 148.048199 550.406903 \n", - "5 92.930400 93.020601 113.056104 122.476104 150.119004 537.076992 \n", - "12 94.695951 94.988002 111.319799 118.210000 134.270997 527.054914 \n", - "4 95.322943 97.896699 117.221999 125.852400 141.615501 523.628160 \n", - "13 96.016296 97.410200 113.779899 120.184499 136.929395 519.810588 \n", - "\n", - " errors \n", - "60 {'OK': 190361, 'Unavailable': 48} \n", - "52 {'OK': 190333, 'Unavailable': 71} \n", - "45 {'OK': 185606, 'Unavailable': 49} \n", - "61 {'OK': 182555, 'Unavailable': 49} \n", - "55 {'OK': 177471, 'Unavailable': 50} \n", - "47 {'OK': 176302, 'Unavailable': 50} \n", - "62 {'OK': 171799, 'Unavailable': 50} \n", - "54 {'OK': 171578, 'Unavailable': 49} \n", - "37 {'OK': 169973, 'Unavailable': 50} \n", - "36 {'OK': 169880, 'Unavailable': 48} \n", - "46 {'OK': 169202, 'Unavailable': 71} \n", - "53 {'OK': 168074, 'Unavailable': 50} \n", - "63 {'OK': 167180, 'Unavailable': 49} \n", - "39 {'OK': 165711, 'Unavailable': 69} \n", - "38 {'OK': 164333, 'Unavailable': 49} \n", - "44 {'OK': 160053, 'Unavailable': 50} \n", - "29 {'OK': 94678, 'Unavailable': 50} \n", - "28 {'OK': 94233, 'Unavailable': 50} \n", - "20 {'OK': 91119, 'Unavailable': 50} \n", - "21 {'OK': 89501, 'Unavailable': 50} \n", - "30 {'OK': 89044, 'Unavailable': 50} \n", - "7 {'OK': 88318, 'Unavailable': 49} \n", - "31 {'OK': 84796, 'Unavailable': 50} \n", - "22 {'OK': 84514, 'Unavailable': 50} \n", - "15 {'OK': 83622, 'Unavailable': 50} \n", - "23 {'OK': 81035, 'Unavailable': 50} \n", - "14 {'OK': 66728, 'Unavailable': 50} \n", - "6 {'OK': 66003, 'Unavailable': 50} \n", - "5 {'OK': 64405, 'Unavailable': 50} \n", - "12 {'OK': 63202, 'Unavailable': 50} \n", - "4 {'OK': 62790, 'Unavailable': 50} \n", - "13 {'OK': 62332, 'Unavailable': 50} " - ] - }, - "execution_count": 191, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df[df[\"apiType\"] == \"grpc\"].sort_values(\"rate\", ascending=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.4" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/batch/benchmarking-argo-workflows/README.md b/examples/batch/benchmarking-argo-workflows/README.md deleted file mode 100644 index 5fab1300cc..0000000000 --- a/examples/batch/benchmarking-argo-workflows/README.md +++ /dev/null @@ -1,1942 +0,0 @@ -## Benchmarking with Argo Worfklows & Vegeta - -In this notebook we will dive into how you can run bench marking with batch processing with Argo Workflows, Seldon Core and Vegeta. - -Dependencies: - -* Seldon core installed as per the docs with Istio as an ingress -* Argo Workfklows installed in cluster (and argo CLI for commands) - - -## Setup - -### Install Seldon Core -Use the notebook to [set-up Seldon Core with Ambassador or Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html). - -Note: If running with KIND you need to make sure do follow [these steps](https://github.com/argoproj/argo/issues/2376#issuecomment-595593237) as workaround to the `/.../docker.sock` known issue. - - -### Install Argo Workflows -You can follow the instructions from the official [Argo Workflows Documentation](https://github.com/argoproj/argo#quickstart). - -You also need to make sure that argo has permissions to create seldon deployments - for this you can just create a default-admin rolebinding as follows: - - -```python -!kubectl create rolebinding default-admin --clusterrole=admin --serviceaccount=default:default -``` - - rolebinding.rbac.authorization.k8s.io/default-admin created - - - -```python -def get_results(results, print_results=True): - final = {} - if "average" in results: - final["mean"] = results["average"] / 1e6 - if results.get("latencyDistribution", False): - final["50th"] = results["latencyDistribution"][-5]["latency"] / 1e6 - final["90th"] = results["latencyDistribution"][-3]["latency"] / 1e6 - final["95th"] = results["latencyDistribution"][-2]["latency"] / 1e6 - final["99th"] = results["latencyDistribution"][-1]["latency"] / 1e6 - final["rate"] = results["rps"] - final["errors"] = results["statusCodeDistribution"] - else: - final["mean"] = results["latencies"]["mean"] / 1e6 - final["50th"] = results["latencies"]["50th"] / 1e6 - final["90th"] = results["latencies"]["90th"] / 1e6 - final["95th"] = results["latencies"]["95th"] / 1e6 - final["99th"] = results["latencies"]["99th"] / 1e6 - final["rate"] = results["throughput"] - final["errors"] = results["errors"] - if print_results: - print("Latencies:") - print("\tmean:", final["mean"], "ms") - print("\t50th:", final["50th"], "ms") - print("\t90th:", final["90th"], "ms") - print("\t95th:", final["95th"], "ms") - print("\t99th:", final["99th"], "ms") - print("") - print("Rate:", str(final["rate"]) + "/s") - print("Errors:", final["errors"]) - return final -``` - -### Create Benchmark Argo Workflow - -In order to create a benchmark, we created a simple argo workflow template so you can leverage the power of the helm charts. - -Before we dive into the contents of the full helm chart, let's first give it a try with some of the settings. - -We will run a batch job that will set up a Seldon Deployment with 1 replicas and 4 cpus (with 100 max workers) to send requests. - - -```python -!helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \ - --set workflow.name=seldon-benchmark-process \ - --set seldonDeployment.name=sklearn \ - --set seldonDeployment.replicas=1 \ - --set seldonDeployment.serverWorkers=1 \ - --set seldonDeployment.serverThreads=10 \ - --set seldonDeployment.apiType=rest \ - --set benchmark.cpus=4 \ - --set benchmark.maxWorkers=100 \ - --set benchmark.duration=30s \ - --set benchmark.rate=0 \ - --set benchmark.data='\{"data": {"ndarray": [[0\,1\,2\,3]]\}\}' \ - | argo submit - -``` - - Name: seldon-benchmark-process - Namespace: default - ServiceAccount: default - Status: Pending - Created: Fri Aug 07 18:09:40 +0100 (now) - - - -```python -!argo list -``` - - NAME STATUS AGE DURATION PRIORITY - seldon-benchmark-process Succeeded 2m 1m 0 - - - -```python -!argo get seldon-benchmark-process -``` - - Name: seldon-benchmark-process - Namespace: default - ServiceAccount: default - Status: Succeeded - Created: Fri Aug 07 18:09:40 +0100 (2 minutes ago) - Started: Fri Aug 07 18:09:40 +0100 (2 minutes ago) - Finished: Fri Aug 07 18:11:09 +0100 (51 seconds ago) - Duration: 1 minute 29 seconds - - STEP PODNAME DURATION MESSAGE - ✔ seldon-benchmark-process (seldon-benchmark-process) - ├---✔ create-seldon-resource (create-seldon-resource-template) seldon-benchmark-process-3980407503 2s - ├---✔ wait-seldon-resource (wait-seldon-resource-template) seldon-benchmark-process-2136965893 49s - └---✔ run-benchmark (run-benchmark-template) seldon-benchmark-process-780051119 32s - - - -```python -!argo logs -w seldon-benchmark-process -``` - - create-seldon-resource: time="2020-08-07T17:09:41.804Z" level=info msg="Starting Workflow Executor" version=v2.9.3 - create-seldon-resource: time="2020-08-07T17:09:41.809Z" level=info msg="Creating a docker executor" - create-seldon-resource: time="2020-08-07T17:09:41.809Z" level=info msg="Executor (version: v2.9.3, build_date: 2020-07-18T19:11:19Z) initialized (pod: default/seldon-benchmark-process-3980407503) with template:\n{\"name\":\"create-seldon-resource-template\",\"arguments\":{},\"inputs\":{},\"outputs\":{},\"metadata\":{},\"resource\":{\"action\":\"create\",\"manifest\":\"apiVersion: machinelearning.seldon.io/v1\\nkind: SeldonDeployment\\nmetadata:\\n name: \\\"sklearn\\\"\\n namespace: default\\n ownerReferences:\\n - apiVersion: argoproj.io/v1alpha1\\n blockOwnerDeletion: true\\n kind: Workflow\\n name: \\\"seldon-benchmark-process\\\"\\n uid: \\\"e0364966-b2c1-4ee7-a7cf-421952ba3c7a\\\"\\nspec:\\n annotations:\\n seldon.io/executor: \\\"false\\\"\\n name: \\\"sklearn\\\"\\n transport: rest\\n predictors:\\n - componentSpecs:\\n - spec:\\n containers:\\n - name: classifier\\n env:\\n - name: GUNICORN_THREADS\\n value: 10\\n - name: GUNICORN_WORKERS\\n value: 1\\n resources:\\n requests:\\n cpu: 50m\\n memory: 100Mi\\n limits:\\n cpu: 50m\\n memory: 1000Mi\\n graph:\\n children: []\\n implementation: SKLEARN_SERVER\\n modelUri: gs://seldon-models/sklearn/iris\\n name: classifier\\n name: default\\n replicas: 1\\n\"}}" - create-seldon-resource: time="2020-08-07T17:09:41.809Z" level=info msg="Loading manifest to /tmp/manifest.yaml" - create-seldon-resource: time="2020-08-07T17:09:41.810Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json" - create-seldon-resource: time="2020-08-07T17:09:42.456Z" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn - create-seldon-resource: time="2020-08-07T17:09:42.457Z" level=info msg="No output parameters" - wait-seldon-resource: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 0 of 1 updated replicas are available... - wait-seldon-resource: deployment "sklearn-default-0-classifier" successfully rolled out - run-benchmark: {"latencies":{"total":3011298973622,"mean":339033885,"50th":272840630,"90th":339539236,"95th":368299307,"99th":4982426813,"max":5597505277,"min":206244298},"bytes_in":{"total":3081764,"mean":346.9673496960144},"bytes_out":{"total":301988,"mean":34},"earliest":"2020-08-07T17:10:37.117884325Z","latest":"2020-08-07T17:11:07.118729145Z","end":"2020-08-07T17:11:07.366654843Z","duration":30000844820,"wait":247925698,"requests":8882,"rate":296.05832946673667,"throughput":293.63176909007353,"success":1,"status_codes":{"200":8882},"errors":[]} - - - -```python -import json -wf_logs = !argo logs -w seldon-benchmark-process -wf_bench = wf_logs[-1] -wf_json_str = wf_bench[24:] -results = json.loads(wf_json_str) - -print("Latencies:") -print("\tmean:", results["latencies"]["mean"] / 1e6, "ms") -print("\t50th:", results["latencies"]["50th"] / 1e6, "ms") -print("\t90th:", results["latencies"]["90th"] / 1e6, "ms") -print("\t95th:", results["latencies"]["95th"] / 1e6, "ms") -print("\t99th:", results["latencies"]["99th"] / 1e6, "ms") -print("") -print("Throughput:", str(results["throughput"]) + "/s") -print("Errors:", len(results["errors"]) > 0) -``` - - Latencies: - mean: 339.033885 ms - 50th: 272.84063 ms - 90th: 339.539236 ms - 95th: 368.299307 ms - 99th: 4982.426813 ms - - Throughput: 293.63176909007353/s - Errors: False - - - -```python -!argo delete seldon-benchmark-process -``` - - Workflow 'seldon-benchmark-process' deleted - - -## Create GRPC benchmark with GHZ and Argo Workflows - - -```python -!helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \ - --set workflow.name=seldon-benchmark-process \ - --set seldonDeployment.name=sklearn \ - --set seldonDeployment.replicas=1 \ - --set seldonDeployment.serverWorkers=1 \ - --set seldonDeployment.serverThreads=10 \ - --set seldonDeployment.apiType=grpc \ - --set benchmark.cpus=4 \ - --set benchmark.maxWorkers=100 \ - --set benchmark.duration="120s" \ - --set benchmark.rate=0 \ - --set benchmark.data='\{"data": {"ndarray": [[0\,1\,2\,3]]\}\}' \ - | argo submit - -``` - - Name: seldon-benchmark-process - Namespace: default - ServiceAccount: default - Status: Pending - Created: Fri Aug 07 18:22:38 +0100 (now) - - - -```python -!argo list -``` - - NAME STATUS AGE DURATION PRIORITY - seldon-benchmark-process Succeeded 4m 2m 0 - - - -```python -!argo get seldon-benchmark-process -``` - - Name: seldon-benchmark-process - Namespace: default - ServiceAccount: default - Status: Succeeded - Created: Fri Aug 07 18:22:38 +0100 (4 minutes ago) - Started: Fri Aug 07 18:22:38 +0100 (4 minutes ago) - Finished: Fri Aug 07 18:25:11 +0100 (1 minute ago) - Duration: 2 minutes 33 seconds - - STEP PODNAME DURATION MESSAGE - ✔ seldon-benchmark-process (seldon-benchmark-process) - ├---✔ create-seldon-resource (create-seldon-resource-template) seldon-benchmark-process-3980407503 2s - ├---✔ wait-seldon-resource (wait-seldon-resource-template) seldon-benchmark-process-2136965893 26s - └---✔ run-benchmark (run-benchmark-template) seldon-benchmark-process-780051119 2m - - - -```python -!argo logs -w seldon-benchmark-process -``` - - create-seldon-resource: time="2020-08-07T17:22:39.446Z" level=info msg="Starting Workflow Executor" version=v2.9.3 - create-seldon-resource: time="2020-08-07T17:22:39.450Z" level=info msg="Creating a docker executor" - create-seldon-resource: time="2020-08-07T17:22:39.450Z" level=info msg="Executor (version: v2.9.3, build_date: 2020-07-18T19:11:19Z) initialized (pod: default/seldon-benchmark-process-3980407503) with template:\n{\"name\":\"create-seldon-resource-template\",\"arguments\":{},\"inputs\":{},\"outputs\":{},\"metadata\":{},\"resource\":{\"action\":\"create\",\"manifest\":\"apiVersion: machinelearning.seldon.io/v1\\nkind: SeldonDeployment\\nmetadata:\\n name: \\\"sklearn\\\"\\n namespace: default\\n ownerReferences:\\n - apiVersion: argoproj.io/v1alpha1\\n blockOwnerDeletion: true\\n kind: Workflow\\n name: \\\"seldon-benchmark-process\\\"\\n uid: \\\"e472d69d-44ed-4a45-86b3-d4b64146002b\\\"\\nspec:\\n name: \\\"sklearn\\\"\\n transport: grpc\\n predictors:\\n - componentSpecs:\\n - spec:\\n containers:\\n - name: classifier\\n env:\\n - name: GUNICORN_THREADS\\n value: 10\\n - name: GUNICORN_WORKERS\\n value: 1\\n graph:\\n children: []\\n implementation: SKLEARN_SERVER\\n modelUri: gs://seldon-models/sklearn/iris\\n name: classifier\\n name: default\\n replicas: 1\\n\"}}" - create-seldon-resource: time="2020-08-07T17:22:39.450Z" level=info msg="Loading manifest to /tmp/manifest.yaml" - create-seldon-resource: time="2020-08-07T17:22:39.450Z" level=info msg="kubectl create -f /tmp/manifest.yaml -o json" - create-seldon-resource: time="2020-08-07T17:22:40.060Z" level=info msg=default/SeldonDeployment.machinelearning.seldon.io/sklearn - create-seldon-resource: time="2020-08-07T17:22:40.060Z" level=info msg="No output parameters" - wait-seldon-resource: Waiting for deployment "sklearn-default-0-classifier" rollout to finish: 0 of 1 updated replicas are available... - wait-seldon-resource: deployment "sklearn-default-0-classifier" successfully rolled out - run-benchmark: {"date":"2020-08-07T17:25:09Z","endReason":"timeout","options":{"host":"istio-ingressgateway.istio-system.svc.cluster.local:80","proto":"/proto/prediction.proto","import-paths":["/proto","."],"call":"seldon.protos.Seldon/Predict","insecure":true,"total":2147483647,"concurrency":50,"connections":1,"duration":120000000000,"timeout":20000000000,"dial-timeout":10000000000,"data":{"data":{"ndarray":[[0,1,2,3]]}},"binary":false,"metadata":{"namespace":"default","seldon":"sklearn"},"CPUs":4},"count":88874,"total":120001033613,"average":67376309,"fastest":21863600,"slowest":148816057,"rps":740.6102874631579,"errorDistribution":{"rpc error: code = Unavailable desc = transport is closing":50},"statusCodeDistribution":{"OK":88824,"Unavailable":50},"latencyDistribution":[{"percentage":10,"latency":54583101},{"percentage":25,"latency":59326600},{"percentage":50,"latency":65257398},{"percentage":75,"latency":73167799},{"percentage":90,"latency":82939600},{"percentage":95,"latency":89598800},{"percentage":99,"latency":101463001}]} - - - -```python -import json -wf_logs = !argo logs -w seldon-benchmark-process -wf_bench = wf_logs[-1] -wf_json_str = wf_bench[24:] -results = json.loads(wf_json_str) - -print("Latencies:") -print("\tmean:", results["average"] / 1e6, "ms") -print("\t50th:", results["latencyDistribution"][-5]["latency"] / 1e6, "ms") -print("\t90th:", results["latencyDistribution"][-3]["latency"] / 1e6, "ms") -print("\t95th:", results["latencyDistribution"][-2]["latency"] / 1e6, "ms") -print("\t99th:", results["latencyDistribution"][-1]["latency"] / 1e6, "ms") -print("") -print("Rate:", str(results["rps"]) + "/s") -print("Errors:", results["statusCodeDistribution"].get("Unavailable", 0) > 0) -print("Errors:", results["statusCodeDistribution"]) -``` - - Latencies: - mean: 67.376309 ms - 50th: 65.257398 ms - 90th: 82.9396 ms - 95th: 89.5988 ms - 99th: 101.463001 ms - - Rate: 740.6102874631579/s - Errors: True - Errors: {'OK': 88824, 'Unavailable': 50} - - - -```python -!argo delete seldon-benchmark-process -``` - - Workflow 'seldon-benchmark-process' deleted - - -## Run a set of tests - -We can now leverage the helm charts we created above to run a grid search on a set of parameters. - - -```python -import itertools as it -import json -import time - -grid_opts = { - "A-replicas": [1, 3], - "B-serverWorkers": [1, 4], - "C-serverThreads": [50, 200], - "D-apiType": ["rest", "grpc"], - "E-cpus": [1, 4], - "F-maxWorkers": [100, 300], - "G-useEngine": ["true", "false"], -} - -allNames = sorted(grid_opts) -combinations = it.product(*(grid_opts[Name] for Name in allNames)) -all_results = [] -for curr_values in combinations: - print("VALUES:", curr_values) - replicas, server_workers, server_threads, api_type, cpus, max_wokers, use_engine = curr_values - - # For some reason python vars don't work with multiline helm charts - %env REPLICAS=$replicas - %env SERVER_WORKERS=$server_workers - %env SERVER_THREADS=$server_threads - %env API_TYPE=$api_type - %env CPUS=$cpus - %env MAX_WORKERS=$max_wokers - %env USE_ENGINE=$use_engine - - !helm template seldon-benchmark-workflow helm-charts/seldon-benchmark-workflow/ \ - --set workflow.name=seldon-benchmark-process \ - --set seldonDeployment.name=sklearn \ - --set seldonDeployment.replicas=$REPLICAS \ - --set seldonDeployment.serverWorkers=$SERVER_WORKERS \ - --set seldonDeployment.serverThreads=$SERVER_THREADS \ - --set seldonDeployment.apiType=$API_TYPE \ - --set seldonDeployment.useEngine=\"$USE_ENGINE\" \ - --set benchmark.cpus=$CPUS \ - --set benchmark.maxWorkers=$MAX_WORKERS \ - --set benchmark.duration=120s \ - --set benchmark.rate=0 \ - --set benchmark.data='\{"data": {"ndarray": [[0\,1\,2\,3]]\}\}' \ - | argo submit --wait - - - !argo wait seldon-benchmark-process - - wf_logs = !argo logs -w seldon-benchmark-process - wf_bench = wf_logs[-1] - wf_json_str = wf_bench[24:] - results = json.loads(wf_json_str) - - result = get_results(results) - result["replicas"] = replicas - result["server_workers"] = server_workers - result["server_threads"] = server_threads - result["apiType"] = api_type - result["cpus"] = cpus - result["max_wokers"] = max_wokers - result["use_engine"] = use_engine - all_results.append(result) - - !argo delete seldon-benchmark-process - time.sleep(1) - print("\n\n") - -``` - -## Deeper Analysis -Now that we have all the parameters, we can do a deeper analysis - - -```python -import pandas as pd -df = pd.DataFrame.from_dict(results) -df.head() -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
01150rest1200true489.269344455.617128612.294382672.510108832.322767407.879172[]
11150rest1200false529.767457514.151876591.278115621.463805749.348556376.649458[]
21150rest4200true547.618426526.472215661.947413720.039676863.596098364.363839[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
41150grpc1200true95.32294397.896699117.221999125.852400141.615501523.628160{'OK': 62790, 'Unavailable': 50}
-
- - - -### GRPC as expected outperforms REST - - -```python -df.sort_values("rate", ascending=False) -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
6034200grpc1200true31.38986123.76958971.58379578.88139891.3127971586.593680{'OK': 190361, 'Unavailable': 48}
523450grpc1200true31.39845126.31300064.84151573.03580088.7441981586.555365{'OK': 190333, 'Unavailable': 71}
4531200grpc1200false32.19124030.44830260.61630168.72440691.4843081547.003054{'OK': 185606, 'Unavailable': 49}
6134200grpc1200false32.72767428.48340063.75079672.59731090.6938121521.590875{'OK': 182555, 'Unavailable': 49}
553450grpc4200false33.62984829.61070167.06589577.77310097.2965991479.320474{'OK': 177471, 'Unavailable': 50}
.............................................
1011200rest4200true571.452398556.699256693.093315751.1975981024.233714348.889260[]
1111200rest4200false587.900216556.869872723.744376774.244702939.994423339.396160[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
811200rest1200true633.043624617.853285741.229073776.5605781846.623159314.908167[]
911200rest1200false641.530606653.922529802.558303847.4144841570.484029310.839312[]
-

64 rows × 14 columns

-
- - - -### Deeper dive REST -As expected replicas has the biggest impact. It seems the parameters on the benchmark worker don't seem to affect throughput. - - -```python -df[df["apiType"]=="rest"].sort_values("rate", ascending=False) -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
4131200rest1200false201.1675468.844305629.250888690.807158809.635194992.298652[]
483450rest1200true208.42957611.377699655.466848685.265506758.664504957.846772[]
513450rest4200false211.22852613.592301641.484819675.713639795.682869945.090980[]
5934200rest4200false214.35883413.573121670.449768690.048496722.537613930.694079[]
5734200rest1200false216.6463209.336961684.733598704.485018733.636276921.350903[]
4031200rest1200true217.72239716.593757657.144743695.158232745.726065916.803160[]
323150rest1200true218.81795210.808913689.809571757.737985867.650689912.589694[]
5634200rest1200true221.0318769.197338690.217169711.800471742.657817903.072311[]
503450rest4200true221.26324916.583482688.637696711.870214781.197685902.315850[]
5834200rest4200true221.56695611.037262685.417461713.923684771.814053901.132352[]
353150rest4200false225.71911415.998348704.701196741.890962852.664830884.187996[]
333150rest1200false229.6533669.844413725.066803775.186525857.762245869.461119[]
4231200rest4200true231.01653615.829218737.382688788.027859885.482116863.960992[]
493450rest1200false231.98692711.193407702.083677769.889421901.360146860.495277[]
4331200rest4200false239.15079414.147647722.982655789.211063929.436195834.381347[]
343150rest4200true240.088790121.078205707.862815771.405571965.932529831.402721[]
2614200rest4200true413.608259409.729690442.576049460.804621502.762769482.699096[]
171450rest1200false429.042835412.423403500.170846522.423418586.685379465.431891[]
2714200rest4200false432.609142426.606234488.443435512.393140556.238288461.578501[]
2514200rest1200false463.422714450.181537551.644801602.270942670.647806430.891782[]
161450rest1200true475.510231456.056479583.716159650.365364746.791628419.975983[]
191450rest4200false481.143061450.734477602.026223689.302618863.072782414.795159[]
181450rest4200true488.185779436.842244628.922397735.5126541068.474298408.992844[]
01150rest1200true489.269344455.617128612.294382672.510108832.322767407.879172[]
2414200rest1200true514.472545488.358257591.629431631.3928131517.062374387.882855[]
11150rest1200false529.767457514.151876591.278115621.463805749.348556376.649458[]
21150rest4200true547.618426526.472215661.947413720.039676863.596098364.363839[]
1011200rest4200true571.452398556.699256693.093315751.1975981024.233714348.889260[]
1111200rest4200false587.900216556.869872723.744376774.244702939.994423339.396160[]
31150rest4200false593.880113602.945695737.993290770.7775431003.510371336.075411[]
811200rest1200true633.043624617.853285741.229073776.5605781846.623159314.908167[]
911200rest1200false641.530606653.922529802.558303847.4144841570.484029310.839312[]
-
- - - -### Deep dive on GRPC - - -```python -df[df["apiType"]=="grpc"].sort_values("rate", ascending=False) -``` - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
replicasserver_workersserver_threadsapiTypecpusmax_wokersuse_enginemean50th90th95th99thrateerrors
6034200grpc1200true31.38986123.76958971.58379578.88139891.3127971586.593680{'OK': 190361, 'Unavailable': 48}
523450grpc1200true31.39845126.31300064.84151573.03580088.7441981586.555365{'OK': 190333, 'Unavailable': 71}
4531200grpc1200false32.19124030.44830260.61630168.72440691.4843081547.003054{'OK': 185606, 'Unavailable': 49}
6134200grpc1200false32.72767428.48340063.75079672.59731090.6938121521.590875{'OK': 182555, 'Unavailable': 49}
553450grpc4200false33.62984829.61070167.06589577.77310097.2965991479.320474{'OK': 177471, 'Unavailable': 50}
4731200grpc4200false33.86102330.20740070.27269883.485103105.6393011469.503585{'OK': 176302, 'Unavailable': 50}
6234200grpc4200true34.74680131.89658572.73279684.03276399.4330901432.045405{'OK': 171799, 'Unavailable': 50}
543450grpc4200true34.78688332.14119772.55431382.64970295.0497051430.209225{'OK': 171578, 'Unavailable': 49}
373150grpc1200false35.14937635.15318762.84280072.79180094.2402991416.745392{'OK': 169973, 'Unavailable': 50}
363150grpc1200true35.16765731.85930065.64489576.24079998.9258991415.967279{'OK': 169880, 'Unavailable': 48}
4631200grpc4200true35.28617324.98850083.07930194.264796111.4488951410.595798{'OK': 169202, 'Unavailable': 71}
533450grpc1200false35.54394030.52890069.44989582.465882100.3811951400.945365{'OK': 168074, 'Unavailable': 50}
6334200grpc4200false35.70618130.17530076.12170185.84238599.0727011393.469861{'OK': 167180, 'Unavailable': 49}
393150grpc4200false36.02680433.54119269.94279881.321704108.5289011381.482907{'OK': 165711, 'Unavailable': 69}
383150grpc4200true36.32571835.59849873.21199782.948302102.2483971369.739820{'OK': 164333, 'Unavailable': 49}
4431200grpc1200true37.32656135.60938870.52259879.731401101.2974001334.058278{'OK': 160053, 'Unavailable': 50}
2914200grpc1200false63.24012961.51920172.90500077.14070089.520499789.347786{'OK': 94678, 'Unavailable': 50}
2814200grpc1200true63.53711961.85520074.29910079.87660197.179900785.631011{'OK': 94233, 'Unavailable': 50}
201450grpc1200true65.71157764.22050078.08530083.56360094.907700759.690398{'OK': 91119, 'Unavailable': 50}
211450grpc1200false66.89814363.42080083.83710092.332400108.138499746.209307{'OK': 89501, 'Unavailable': 50}
3014200grpc4200true67.21160965.50420079.98989986.808200106.460500742.433252{'OK': 89044, 'Unavailable': 50}
71150grpc4200false67.77063262.16850488.674303102.537000120.848185736.385539{'OK': 88318, 'Unavailable': 49}
3114200grpc4200false70.57783468.97289984.86920089.875600102.761897707.046156{'OK': 84796, 'Unavailable': 50}
221450grpc4200true70.81841167.59160087.91410497.004000115.388900704.647865{'OK': 84514, 'Unavailable': 50}
1511200grpc4200false71.57162769.34870091.60959898.471998111.237797697.252435{'OK': 83622, 'Unavailable': 50}
231450grpc4200false73.85378070.60470191.03140098.064600116.658902675.704389{'OK': 81035, 'Unavailable': 50}
1411200grpc4200true89.66250087.678702107.762199118.226099146.838610556.478774{'OK': 66728, 'Unavailable': 50}
61150grpc4200true90.65502591.964500108.453597116.581800148.048199550.406903{'OK': 66003, 'Unavailable': 50}
51150grpc1200false92.93040093.020601113.056104122.476104150.119004537.076992{'OK': 64405, 'Unavailable': 50}
1211200grpc1200true94.69595194.988002111.319799118.210000134.270997527.054914{'OK': 63202, 'Unavailable': 50}
41150grpc1200true95.32294397.896699117.221999125.852400141.615501523.628160{'OK': 62790, 'Unavailable': 50}
1311200grpc1200false96.01629697.410200113.779899120.184499136.929395519.810588{'OK': 62332, 'Unavailable': 50}
-
- - - - -```python - -``` diff --git a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml b/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml deleted file mode 100644 index 40b65bf491..0000000000 --- a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml +++ /dev/null @@ -1,123 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - {{- if eq .Values.workflow.useNameAsGenerateName "false" }} - name: {{ .Values.workflow.name }} - {{- else }} - generateName: {{ .Values.workflow.name }} - {{- end }} - namespace: {{ .Values.workflow.namespace }} -spec: - entrypoint: seldon-benchmark-process - templates: - - name: seldon-benchmark-process - steps: - - - name: create-seldon-resource - template: create-seldon-resource-template - - - name: wait-seldon-resource - template: wait-seldon-resource-template - - - name: run-benchmark - template: run-benchmark-template - - - name: create-seldon-resource-template - resource: - action: create - manifest: | - apiVersion: machinelearning.seldon.io/v1 - kind: SeldonDeployment - metadata: - name: "{{ .Values.seldonDeployment.name }}" - namespace: {{ .Values.workflow.namespace }} - ownerReferences: - - apiVersion: argoproj.io/v1alpha1 - blockOwnerDeletion: true - kind: Workflow - name: "{{`{{workflow.name}}`}}" - uid: "{{`{{workflow.uid}}`}}" - spec: - {{- if eq .Values.seldonDeployment.useEngine "true" }} - annotations: - seldon.io/executor: "false" - {{- end }} - name: "{{ .Values.seldonDeployment.name }}" - transport: {{ .Values.seldonDeployment.apiType }} - predictors: - - componentSpecs: - - spec: - containers: - - name: classifier - env: - - name: GUNICORN_THREADS - value: {{ .Values.seldonDeployment.serverThreads }} - - name: GUNICORN_WORKERS - value: {{ .Values.seldonDeployment.serverWorkers }} - {{- if eq .Values.seldonDeployment.enableResources "true" }} - resources: - requests: - cpu: {{ .Values.seldonDeployment.requests.cpu }} - memory: {{ .Values.seldonDeployment.requests.memory }} - limits: - cpu: {{ .Values.seldonDeployment.limits.cpu }} - memory: {{ .Values.seldonDeployment.limits.memory }} - {{- end }} - graph: - children: [] - implementation: {{ .Values.seldonDeployment.server }} - modelUri: {{ .Values.seldonDeployment.modelUri }} - name: classifier - name: default - replicas: {{ .Values.seldonDeployment.replicas }} - - - name: wait-seldon-resource-template - script: - image: bitnami/kubectl:1.17 - command: [bash] - source: | - sleep {{ .Values.seldonDeployment.waitTime }} - kubectl rollout status \ - deploy/$(kubectl get deploy -l seldon-deployment-id="{{ .Values.seldonDeployment.name }}" -o jsonpath='{.items[0].metadata.name}') - - - name: run-benchmark-template - script: - {{- if .Values.benchmark.imageOverride }} - image: "{{ .Values.benchmark.imageOverride }}" - {{- else if eq .Values.seldonDeployment.apiType "rest" }} - image: peterevans/vegeta:6.8.1 - {{- else }} - image: seldonio/ghz:v0.55.0 - {{- end }} - command: [sh] - {{- if eq .Values.seldonDeployment.apiType "rest" }} - source: | - echo '{"method": "POST", "header": {"Content-Type": ["application/json"] }, "url": "http://{{ .Values.benchmark.host }}/seldon/{{ .Values.workflow.namespace }}/{{ .Values.seldonDeployment.name }}/api/v1.0/predictions", "body": "{{ .Values.benchmark.data | b64enc }}" }' \ - | vegeta \ - -cpus={{ .Values.benchmark.cpus }} \ - attack \ - -duration={{ .Values.benchmark.duration }} \ - -rate={{ .Values.benchmark.rate }} \ - -max-workers={{ .Values.benchmark.maxWorkers }} \ - -format=json \ - | vegeta \ - report \ - -type=json - - {{- else }} - source: | - ghz \ - --insecure \ - --proto /proto/prediction.proto \ - --call seldon.protos.Seldon/Predict \ - --data='{{ .Values.benchmark.data }}' \ - --qps={{ .Values.benchmark.rate }} \ - --cpus={{ .Values.benchmark.cpus }} \ - --duration="{{ .Values.benchmark.duration }}" \ - --format json \ - --metadata='{"seldon": "{{ .Values.seldonDeployment.name }}", "namespace": "{{ .Values.workflow.namespace }}"}' \ - {{ .Values.benchmark.host }} \ - | jq \ - -c \ - 'del(.histogram)|del(.details)' - - {{- end }} - diff --git a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/Chart.yaml b/helm-charts/seldon-benchmark-workflow/Chart.yaml similarity index 100% rename from examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/Chart.yaml rename to helm-charts/seldon-benchmark-workflow/Chart.yaml diff --git a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/README.md b/helm-charts/seldon-benchmark-workflow/README.md similarity index 100% rename from examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/README.md rename to helm-charts/seldon-benchmark-workflow/README.md diff --git a/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml b/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml new file mode 100644 index 0000000000..9c866de1df --- /dev/null +++ b/helm-charts/seldon-benchmark-workflow/templates/workflow.yaml @@ -0,0 +1,450 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + {{- if eq .Values.workflow.useNameAsGenerateName "false" }} + name: {{ .Values.workflow.name }} + {{- else }} + generateName: {{ .Values.workflow.name }} + {{- end }} + namespace: {{ .Values.workflow.namespace }} +spec: + entrypoint: seldon-benchmark-process + templates: + - name: seldon-benchmark-process + parallelism: {{ .Values.workflow.parallelism }} + steps: + - - name: generate-parameters + template: generate-parameters-template + - - name: run-benchmark-iteration + arguments: + parameters: + - name: listitem + value: "{{`{{item.name}}`}}" + - name: replicas + value: "{{`{{item.replicas}}`}}" + - name: serverWorkers + value: "{{`{{item.serverWorkers}}`}}" + - name: serverThreads + value: "{{`{{item.serverThreads}}`}}" + - name: modelUri + value: "{{`{{item.modelUri}}`}}" + - name: image + value: "{{`{{item.image}}`}}" + - name: server + value: "{{`{{item.server}}`}}" + - name: apiType + value: "{{`{{item.apiType}}`}}" + - name: requestsCpu + value: "{{`{{item.requestsCpu}}`}}" + - name: requestsMemory + value: "{{`{{item.requestsMemory}}`}}" + - name: limitsCpu + value: "{{`{{item.limitsCpu}}`}}" + - name: limitsMemory + value: "{{`{{item.limitsMemory}}`}}" + - name: benchmarkCpu + value: "{{`{{item.benchmarkCpu}}`}}" + - name: concurrency + value: "{{`{{item.concurrency}}`}}" + - name: duration + value: "{{`{{item.duration}}`}}" + - name: rate + value: "{{`{{item.rate}}`}}" + - name: params + value: "{{`{{item.params}}`}}" + - name: disableOrchestrator + value: "{{`{{item.disableOrchestrator}}`}}" + withParam: "{{`{{steps.generate-parameters.outputs.result}}`}}" + template: run-benchmark-iteration-step-template + + - name: run-benchmark-iteration-step-template + inputs: + parameters: + - name: listitem + - name: replicas + - name: serverWorkers + - name: serverThreads + - name: modelUri + - name: image + - name: server + - name: apiType + - name: requestsCpu + - name: requestsMemory + - name: limitsCpu + - name: limitsMemory + - name: benchmarkCpu + - name: concurrency + - name: duration + - name: rate + - name: params + - name: disableOrchestrator + steps: + - - name: create-seldon-resource + template: create-seldon-resource-template + arguments: + parameters: + - name: inparam + value: "{{`{{inputs.parameters.listitem}}`}}" + - name: replicas + value: "{{`{{inputs.parameters.replicas}}`}}" + - name: serverWorkers + value: "{{`{{inputs.parameters.serverWorkers}}`}}" + - name: serverThreads + value: "{{`{{inputs.parameters.serverThreads}}`}}" + - name: modelUri + value: "{{`{{inputs.parameters.modelUri}}`}}" + - name: image + value: "{{`{{inputs.parameters.image}}`}}" + - name: server + value: "{{`{{inputs.parameters.server}}`}}" + - name: apiType + value: "{{`{{inputs.parameters.apiType}}`}}" + - name: requestsCpu + value: "{{`{{inputs.parameters.requestsCpu}}`}}" + - name: requestsMemory + value: "{{`{{inputs.parameters.requestsMemory}}`}}" + - name: limitsCpu + value: "{{`{{inputs.parameters.limitsCpu}}`}}" + - name: limitsMemory + value: "{{`{{inputs.parameters.limitsMemory}}`}}" + - name: benchmarkCpu + value: "{{`{{inputs.parameters.benchmarkCpu}}`}}" + - name: concurrency + value: "{{`{{inputs.parameters.concurrency}}`}}" + - name: duration + value: "{{`{{inputs.parameters.duration}}`}}" + - name: rate + value: "{{`{{inputs.parameters.rate}}`}}" + - name: params + value: "{{`{{inputs.parameters.params}}`}}" + - name: disableOrchestrator + value: "{{`{{inputs.parameters.disableOrchestrator}}`}}" + - - name: wait-seldon-resource + template: wait-seldon-resource-template + arguments: + parameters: + - name: inparam + value: "{{`{{inputs.parameters.listitem}}`}}" + - - name: run-benchmark-rest + template: run-benchmark-template-rest + when: "{{`{{inputs.parameters.apiType}}`}} == rest" + arguments: + parameters: + - name: inparam + value: "{{`{{inputs.parameters.listitem}}`}}" + - name: replicas + value: "{{`{{inputs.parameters.replicas}}`}}" + - name: serverWorkers + value: "{{`{{inputs.parameters.serverWorkers}}`}}" + - name: serverThreads + value: "{{`{{inputs.parameters.serverThreads}}`}}" + - name: modelUri + value: "{{`{{inputs.parameters.modelUri}}`}}" + - name: image + value: "{{`{{inputs.parameters.image}}`}}" + - name: server + value: "{{`{{inputs.parameters.server}}`}}" + - name: apiType + value: "{{`{{inputs.parameters.apiType}}`}}" + - name: requestsCpu + value: "{{`{{inputs.parameters.requestsCpu}}`}}" + - name: requestsMemory + value: "{{`{{inputs.parameters.requestsMemory}}`}}" + - name: limitsCpu + value: "{{`{{inputs.parameters.limitsCpu}}`}}" + - name: limitsMemory + value: "{{`{{inputs.parameters.limitsMemory}}`}}" + - name: benchmarkCpu + value: "{{`{{inputs.parameters.benchmarkCpu}}`}}" + - name: concurrency + value: "{{`{{inputs.parameters.concurrency}}`}}" + - name: duration + value: "{{`{{inputs.parameters.duration}}`}}" + - name: rate + value: "{{`{{inputs.parameters.rate}}`}}" + - name: params + value: "{{`{{inputs.parameters.params}}`}}" + - name: disableOrchestrator + value: "{{`{{inputs.parameters.disableOrchestrator}}`}}" + - name: run-benchmark-grpc + template: run-benchmark-template-grpc + when: "{{`{{inputs.parameters.apiType}}`}} == grpc" + arguments: + parameters: + - name: inparam + value: "{{`{{inputs.parameters.listitem}}`}}" + - name: replicas + value: "{{`{{inputs.parameters.replicas}}`}}" + - name: serverWorkers + value: "{{`{{inputs.parameters.serverWorkers}}`}}" + - name: serverThreads + value: "{{`{{inputs.parameters.serverThreads}}`}}" + - name: modelUri + value: "{{`{{inputs.parameters.modelUri}}`}}" + - name: image + value: "{{`{{inputs.parameters.image}}`}}" + - name: server + value: "{{`{{inputs.parameters.server}}`}}" + - name: apiType + value: "{{`{{inputs.parameters.apiType}}`}}" + - name: requestsCpu + value: "{{`{{inputs.parameters.requestsCpu}}`}}" + - name: requestsMemory + value: "{{`{{inputs.parameters.requestsMemory}}`}}" + - name: limitsCpu + value: "{{`{{inputs.parameters.limitsCpu}}`}}" + - name: limitsMemory + value: "{{`{{inputs.parameters.limitsMemory}}`}}" + - name: benchmarkCpu + value: "{{`{{inputs.parameters.benchmarkCpu}}`}}" + - name: concurrency + value: "{{`{{inputs.parameters.concurrency}}`}}" + - name: duration + value: "{{`{{inputs.parameters.duration}}`}}" + - name: rate + value: "{{`{{inputs.parameters.rate}}`}}" + - name: params + value: "{{`{{inputs.parameters.params}}`}}" + - name: disableOrchestrator + value: "{{`{{inputs.parameters.disableOrchestrator}}`}}" + - - name: delete-seldon-resource + template: delete-seldon-resource-template + arguments: + parameters: + - name: inparam + value: "{{`{{inputs.parameters.listitem}}`}}" + + - name: generate-parameters-template + script: + image: python:alpine3.6 + command: [python] + source: | + import json + import sys + from itertools import product + + delim = "{{ .Values.workflow.paramDelimiter }}" + + params = product( + "{{ .Values.seldonDeployment.replicas }}".split(delim), + "{{ .Values.seldonDeployment.serverWorkers }}".split(delim), + "{{ .Values.seldonDeployment.serverThreads }}".split(delim), + "{{ .Values.seldonDeployment.modelUri }}".split(delim), + "{{ .Values.seldonDeployment.image }}".split(delim), + "{{ .Values.seldonDeployment.server }}".split(delim), + "{{ .Values.seldonDeployment.apiType }}".split(delim), + "{{ .Values.seldonDeployment.requests.cpu }}".split(delim), + "{{ .Values.seldonDeployment.requests.memory }}".split(delim), + "{{ .Values.seldonDeployment.limits.cpu }}".split(delim), + "{{ .Values.seldonDeployment.limits.memory }}".split(delim), + "{{ .Values.benchmark.cpu }}".split(delim), + "{{ .Values.benchmark.concurrency }}".split(delim), + "{{ .Values.benchmark.duration }}".split(delim), + "{{ .Values.benchmark.rate }}".split(delim), + "{{ .Values.seldonDeployment.disableOrchestrator }}".split(delim), + ) + + + list_params = [] + for idx, param in enumerate(params): + name = "{{ .Values.seldonDeployment.name }}-" + str(idx) + curr = { + "name": name, + "replicas": param[0], + "serverWorkers": param[1], + "serverThreads": param[2], + "modelUri": param[3], + "image": param[4], + "server": param[5], + "apiType": param[6], + "requestsCpu": param[7], + "requestsMemory": param[8], + "limitsCpu": param[9], + "limitsMemory": param[10], + "benchmarkCpu": param[11], + "concurrency": param[12], + "duration": param[13], + "rate": param[14], + "disableOrchestrator": param[15], + } + curr["params"] = json.dumps(curr) + list_params.append(curr) + + json.dump(list_params, sys.stdout) + + - name: create-seldon-resource-template + inputs: + parameters: + - name: inparam + - name: replicas + - name: serverWorkers + - name: serverThreads + - name: modelUri + - name: image + - name: server + - name: apiType + - name: requestsCpu + - name: requestsMemory + - name: limitsCpu + - name: limitsMemory + - name: benchmarkCpu + - name: concurrency + - name: duration + - name: rate + - name: params + - name: disableOrchestrator + resource: + action: create + manifest: | + apiVersion: machinelearning.seldon.io/v1 + kind: SeldonDeployment + metadata: + name: "{{`{{inputs.parameters.inparam}}`}}" + namespace: {{ .Values.workflow.namespace }} + ownerReferences: + - apiVersion: argoproj.io/v1alpha1 + blockOwnerDeletion: true + kind: Workflow + name: "{{`{{workflow.name}}`}}" + uid: "{{`{{workflow.uid}}`}}" + spec: + name: "{{`{{inputs.parameters.inparam}}`}}" + transport: "{{`{{inputs.parameters.apiType}}`}}" + predictors: + - annotations: + seldonio/no-engine: "{{`{{inputs.parameters.disableOrchestrator}}`}}" + componentSpecs: + - spec: + containers: + - name: classifier + {{- if .Values.benchmark.image }} + image: "{{`{{inputs.parameters.image}}`}}" + {{- end }} + env: + - name: GUNICORN_THREADS + value: "{{`{{inputs.parameters.serverThreads}}`}}" + - name: GUNICORN_WORKERS + value: "{{`{{inputs.parameters.serverWorkers}}`}}" + {{- if eq .Values.seldonDeployment.enableResources "true" }} + resources: + requests: + cpu: {{`{{inputs.parameters.requestsCpu}}`}} + memory: {{`{{inputs.parameters.requestsMemory}}`}} + limits: + cpu: {{`{{inputs.parameters.limitsCpu}}`}} + memory: {{`{{inputs.parameters.limitsMemory}}`}} + {{- end }} + graph: + children: [] + implementation: {{`{{inputs.parameters.server}}`}} + {{- if .Values.seldonDeployment.modelUri }} + modelUri: {{`{{inputs.parameters.modelUri}}`}} + {{- end }} + name: classifier + name: default + replicas: {{`{{inputs.parameters.replicas}}`}} + + - name: wait-seldon-resource-template + inputs: + parameters: + - name: inparam + script: + image: bitnami/kubectl:1.17 + command: [bash] + source: | + sleep {{ .Values.seldonDeployment.waitTime }} + kubectl rollout status \ + deploy/$(kubectl get deploy -l seldon-deployment-id="{{`{{inputs.parameters.inparam}}`}}" -o jsonpath='{.items[0].metadata.name}') + + - name: run-benchmark-template-rest + inputs: + parameters: + - name: inparam + - name: replicas + - name: serverWorkers + - name: serverThreads + - name: modelUri + - name: image + - name: server + - name: apiType + - name: requestsCpu + - name: requestsMemory + - name: limitsCpu + - name: limitsMemory + - name: benchmarkCpu + - name: concurrency + - name: duration + - name: rate + - name: params + - name: disableOrchestrator + script: + image: {{ .Values.benchmark.restImage }} + command: [sh] + source: | + echo '{"method": "POST", "header": {"Content-Type": ["application/json"] }, "url": "http://{{ .Values.benchmark.host }}/seldon/{{ .Values.workflow.namespace }}/{{`{{inputs.parameters.inparam}}`}}/api/v1.0/predictions", "body": "{{ .Values.benchmark.data | b64enc }}" }' \ + | vegeta \ + -cpus="{{`{{inputs.parameters.benchmarkCpu}}`}}" \ + attack \ + -duration="{{`{{inputs.parameters.duration}}`}}" \ + -rate="{{`{{inputs.parameters.rate}}`}}" \ + -max-connections="{{`{{inputs.parameters.concurrency}}`}}" \ + -max-workers="{{`{{inputs.parameters.concurrency}}`}}" \ + -format=json \ + | vegeta \ + report \ + -type=json \ + | jq -c ". += {\"params\": {{`{{inputs.parameters.params}}`}}}" + + - name: run-benchmark-template-grpc + inputs: + parameters: + - name: inparam + - name: replicas + - name: serverWorkers + - name: serverThreads + - name: modelUri + - name: image + - name: server + - name: apiType + - name: requestsCpu + - name: requestsMemory + - name: limitsCpu + - name: limitsMemory + - name: benchmarkCpu + - name: concurrency + - name: duration + - name: rate + - name: params + - name: disableOrchestrator + script: + image: {{ .Values.benchmark.grpcImage }} + command: [sh] + source: | + ghz \ + --insecure \ + --proto /proto/prediction.proto \ + --call seldon.protos.Seldon/Predict \ + --data='{{ .Values.benchmark.data }}' \ + --qps="{{`{{inputs.parameters.rate}}`}}" \ + --cpus="{{`{{inputs.parameters.benchmarkCpu}}`}}" \ + --duration="{{`{{inputs.parameters.duration}}`}}" \ + --concurrency="{{`{{inputs.parameters.concurrency}}`}}" \ + --format json \ + --metadata='{"seldon": "{{`{{inputs.parameters.inparam}}`}}", "namespace": "{{ .Values.workflow.namespace }}"}' \ + {{ .Values.benchmark.host }} \ + | jq \ + -c \ + 'del(.histogram)|del(.details)' \ + | jq -c ". += {\"params\": {{`{{inputs.parameters.params}}`}}}" + + - name: delete-seldon-resource-template + inputs: + parameters: + - name: inparam + script: + image: bitnami/kubectl:1.17 + command: [bash] + source: | + kubectl delete sdep {{`{{inputs.parameters.inparam}}`}} -n {{ .Values.workflow.namespace }} + diff --git a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/values.yaml b/helm-charts/seldon-benchmark-workflow/values.yaml similarity index 69% rename from examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/values.yaml rename to helm-charts/seldon-benchmark-workflow/values.yaml index 5ec0f25794..70d75be43d 100644 --- a/examples/batch/benchmarking-argo-workflows/helm-charts/seldon-benchmark-workflow/values.yaml +++ b/helm-charts/seldon-benchmark-workflow/values.yaml @@ -5,27 +5,30 @@ workflow: useNameAsGenerateName: "false" # Namespace where to create the workflow and all resources in benchmark job namespace: default + # The number of benchmarks being carried out at the same time + parallelism: 1 + # The default delimiter to use for splitting the parameters provided + paramDelimiter: "|" seldonDeployment: # Name to use for the seldon deployment which by default appends generated workflow ID name: seldon-{{workflow.uid}} - # Image to use for the benchmark client - image: seldonio/seldon-core-s2i-python37:1.3.0-dev + # TODO: Ensure one of image or server is set, and if server is set that modeluri is provided + # Image parameter to use in addition or alternative to prepackaged server + image: # Prepackaged model server to use [see https://docs.seldon.io/projects/seldon-core/en/latest/servers/overview.html] - server: SKLEARN_SERVER + server: CUSTOM_SERVER # The URL for the model that is to be used - modelUri: gs://seldon-models/sklearn/iris + modelUri: # The API Type (REST vs GRPC) apiType: rest - # Whether to use the depricated engine - useEngine: "false" # The number of seldon deployment replicas to launch replicas: 2 # Waiting time before checks for deployment to ensure kubernetes cluster registers create waitTime: 5 # The number of threads spawned by Python Gunicorn Flask server - serverThreads: 10 + serverThreads: 1 # The number of workers spawned by Python Gunicorn Flask server - serverWorkers: 1 + serverWorkers: 4 # Whether to enable resources enableResources: "false" requests: @@ -38,19 +41,25 @@ seldonDeployment: cpu: 50m # Limits for memory (only added if enableResources is enabled) memory: 1000Mi + # Whether to disable service orchestrator to test latency (false by default) + disableOrchestrator: false # The benchmark worker is the component that will send the requests from the files benchmark: # Endpoint of for the benchmark client to contact the seldon deployment host: istio-ingressgateway.istio-system.svc.cluster.local:80 # Number of parallel benchmark client workers to process the data - cpus: 4 + cpu: 4 # Maximum number of workers to allocate for a benchmark - maxWorkers: 100 + concurrency: 1 # Duration of benchmark duration: 30s # Rate (number of requests per second [0 = infinity]) rate: 0 - # Image to use for benchmark - imageOverride: # Data that the benchmark worker will use to send + # TODO: Explore adding multiple data instances with configurable split token (eg |) data: '{"data": {"ndarray": [[0,1,2,3]]}}' + # Image to use for the benchmark REST client + # TODO: Move image to seldonio + restImage: peterevans/vegeta:latest-vegeta12.8.4 + # Image to use for the benchmark GRPC client + grpcImage: seldonio/ghz:v0.55.0 diff --git a/examples/batch/benchmarking-argo-workflows/.gitignore b/testing/benchmarking/automated-benchmark/.gitignore similarity index 100% rename from examples/batch/benchmarking-argo-workflows/.gitignore rename to testing/benchmarking/automated-benchmark/.gitignore diff --git a/testing/benchmarking/automated-benchmark/README.ipynb b/testing/benchmarking/automated-benchmark/README.ipynb new file mode 100644 index 0000000000..345127385b --- /dev/null +++ b/testing/benchmarking/automated-benchmark/README.ipynb @@ -0,0 +1,495 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Benchmarking with Argo Worfklows & Vegeta\n", + "\n", + "In this notebook we will dive into how you can run bench marking with batch processing with Argo Workflows, Seldon Core and Vegeta.\n", + "\n", + "Dependencies:\n", + "\n", + "* Seldon core installed as per the docs with Istio as an ingress \n", + "* Argo Workfklows installed in cluster (and argo CLI for commands)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "### Install Seldon Core\n", + "Use the notebook to [set-up Seldon Core with Ambassador or Istio Ingress](https://docs.seldon.io/projects/seldon-core/en/latest/examples/seldon_core_setup.html).\n", + "\n", + "Note: If running with KIND you need to make sure do follow [these steps](https://github.com/argoproj/argo/issues/2376#issuecomment-595593237) as workaround to the `/.../docker.sock` known issue.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install Argo Workflows\n", + "You can follow the instructions from the official [Argo Workflows Documentation](https://github.com/argoproj/argo#quickstart).\n", + "\n", + "Download the right CLi for your environment following the documentation (https://github.com/argoproj/argo-workflows/releases/tag/v3.0.8)\n", + "\n", + "You also need to make sure that argo has permissions to create seldon deployments - for this you can just create a default-admin rolebinding as follows:" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up the RBAC so the argo workflow is able to create seldon deployments." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Set up the configmap in order for it to work in KIND and other environments where Docker may not be thr main runtime (see https://github.com/argoproj/argo-workflows/issues/5243#issuecomment-792993742)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create Benchmark Argo Workflow\n", + "\n", + "In order to create a benchmark, we created a simple argo workflow template so you can leverage the power of the helm charts.\n", + "\n", + "Before we dive into the contents of the full helm chart, let's first give it a try with some of the settings.\n", + "\n", + "We will run a batch job that will set up a Seldon Deployment with 1 replicas and 4 cpus (with 100 max workers) to send requests." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: seldon-benchmark-process\r\n", + "Namespace: argo\r\n", + "ServiceAccount: default\r\n", + "Status: Pending\r\n", + "Created: Mon Jun 28 18:38:12 +0100 (now)\r\n", + "Progress: \r\n" + ] + } + ], + "source": [ + "!helm template seldon-benchmark-workflow ../../../helm-charts/seldon-benchmark-workflow/ \\\n", + " --set workflow.namespace=argo \\\n", + " --set workflow.name=seldon-benchmark-process \\\n", + " --set workflow.parallelism=2 \\\n", + " --set seldonDeployment.name=sklearn \\\n", + " --set seldonDeployment.replicas=\"1\" \\\n", + " --set seldonDeployment.serverWorkers=\"5\" \\\n", + " --set seldonDeployment.serverThreads=1 \\\n", + " --set seldonDeployment.modelUri=\"gs://seldon-models/sklearn/iris\" \\\n", + " --set seldonDeployment.server=\"SKLEARN_SERVER\" \\\n", + " --set seldonDeployment.apiType=\"rest|grpc\" \\\n", + " --set seldonDeployment.requests.cpu=\"2000Mi\" \\\n", + " --set seldonDeployment.limits.cpu=\"2000Mi\" \\\n", + " --set seldonDeployment.disableOrchestrator=\"true|false\" \\\n", + " --set benchmark.cpu=\"2\" \\\n", + " --set benchmark.concurrency=\"1\" \\\n", + " --set benchmark.duration=\"30s\" \\\n", + " --set benchmark.rate=0 \\\n", + " --set benchmark.data='\\{\"data\": {\"ndarray\": [[0\\,1\\,2\\,3]]\\}\\}' \\\n", + " | argo submit -" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME STATUS AGE DURATION PRIORITY\r\n", + "seldon-benchmark-process Running 20s 20s 0\r\n" + ] + } + ], + "source": [ + "!argo list -n argo" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mseldon-benchmark-process-635956972: [{\"name\": \"sklearn-0\", \"replicas\": \"1\", \"serverWorkers\": \"5\", \"serverThreads\": \"1\", \"modelUri\": \"gs://seldon-models/sklearn/iris\", \"image\": \"\", \"server\": \"SKLEARN_SERVER\", \"apiType\": \"rest\", \"requestsCpu\": \"2000Mi\", \"requestsMemory\": \"100Mi\", \"limitsCpu\": \"2000Mi\", \"limitsMemory\": \"1000Mi\", \"benchmarkCpu\": \"2\", \"concurrency\": \"1\", \"duration\": \"30s\", \"rate\": \"0\", \"disableOrchestrator\": \"true\", \"params\": \"{\\\"name\\\": \\\"sklearn-0\\\", \\\"replicas\\\": \\\"1\\\", \\\"serverWorkers\\\": \\\"5\\\", \\\"serverThreads\\\": \\\"1\\\", \\\"modelUri\\\": \\\"gs://seldon-models/sklearn/iris\\\", \\\"image\\\": \\\"\\\", \\\"server\\\": \\\"SKLEARN_SERVER\\\", \\\"apiType\\\": \\\"rest\\\", \\\"requestsCpu\\\": \\\"2000Mi\\\", \\\"requestsMemory\\\": \\\"100Mi\\\", \\\"limitsCpu\\\": \\\"2000Mi\\\", \\\"limitsMemory\\\": \\\"1000Mi\\\", \\\"benchmarkCpu\\\": \\\"2\\\", \\\"concurrency\\\": \\\"1\\\", \\\"duration\\\": \\\"30s\\\", \\\"rate\\\": \\\"0\\\", \\\"disableOrchestrator\\\": \\\"true\\\"}\"}]\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.287Z\" level=info msg=\"Starting Workflow Executor\" version=\"{v3.0.3 2021-05-11T21:14:20Z 02071057c082cf295ab8da68f1b2027ff8762b5a v3.0.3 clean go1.15.7 gc linux/amd64}\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.289Z\" level=info msg=\"Creating a docker executor\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.289Z\" level=info msg=\"Executor (version: v3.0.3, build_date: 2021-05-11T21:14:20Z) initialized (pod: argo/seldon-benchmark-process-2323867814) with template:\\n{\\\"name\\\":\\\"create-seldon-resource-template\\\",\\\"inputs\\\":{\\\"parameters\\\":[{\\\"name\\\":\\\"inparam\\\",\\\"value\\\":\\\"sklearn-0\\\"},{\\\"name\\\":\\\"replicas\\\",\\\"value\\\":\\\"1\\\"},{\\\"name\\\":\\\"serverWorkers\\\",\\\"value\\\":\\\"5\\\"},{\\\"name\\\":\\\"serverThreads\\\",\\\"value\\\":\\\"1\\\"},{\\\"name\\\":\\\"modelUri\\\",\\\"value\\\":\\\"gs://seldon-models/sklearn/iris\\\"},{\\\"name\\\":\\\"image\\\",\\\"value\\\":\\\"\\\"},{\\\"name\\\":\\\"server\\\",\\\"value\\\":\\\"SKLEARN_SERVER\\\"},{\\\"name\\\":\\\"apiType\\\",\\\"value\\\":\\\"rest\\\"},{\\\"name\\\":\\\"requestsCpu\\\",\\\"value\\\":\\\"2000Mi\\\"},{\\\"name\\\":\\\"requestsMemory\\\",\\\"value\\\":\\\"100Mi\\\"},{\\\"name\\\":\\\"limitsCpu\\\",\\\"value\\\":\\\"2000Mi\\\"},{\\\"name\\\":\\\"limitsMemory\\\",\\\"value\\\":\\\"1000Mi\\\"},{\\\"name\\\":\\\"benchmarkCpu\\\",\\\"value\\\":\\\"2\\\"},{\\\"name\\\":\\\"concurrency\\\",\\\"value\\\":\\\"1\\\"},{\\\"name\\\":\\\"duration\\\",\\\"value\\\":\\\"30s\\\"},{\\\"name\\\":\\\"rate\\\",\\\"value\\\":\\\"0\\\"},{\\\"name\\\":\\\"params\\\",\\\"value\\\":\\\"{\\\\\\\\\\\\\\\"name\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"sklearn-0\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"replicas\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"1\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"serverWorkers\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"5\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"serverThreads\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"1\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"modelUri\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"gs://seldon-models/sklearn/iris\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"image\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"server\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"SKLEARN_SERVER\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"apiType\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"rest\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"requestsCpu\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2000Mi\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"requestsMemory\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"100Mi\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"limitsCpu\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2000Mi\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"limitsMemory\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"1000Mi\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"benchmarkCpu\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"concurrency\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"1\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"duration\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"30s\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"rate\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"0\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"disableOrchestrator\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"true\\\\\\\\\\\\\\\"}\\\"},{\\\"name\\\":\\\"disableOrchestrator\\\",\\\"value\\\":\\\"true\\\"}]},\\\"outputs\\\":{},\\\"metadata\\\":{},\\\"resource\\\":{\\\"action\\\":\\\"create\\\",\\\"manifest\\\":\\\"apiVersion: machinelearning.seldon.io/v1\\\\nkind: SeldonDeployment\\\\nmetadata:\\\\n name: \\\\\\\"sklearn-0\\\\\\\"\\\\n namespace: argo\\\\n ownerReferences:\\\\n - apiVersion: argoproj.io/v1alpha1\\\\n blockOwnerDeletion: true\\\\n kind: Workflow\\\\n name: \\\\\\\"seldon-benchmark-process\\\\\\\"\\\\n uid: \\\\\\\"8be89da7-cb46-40c5-98ac-c17dd9d99d12\\\\\\\"\\\\nspec:\\\\n name: \\\\\\\"sklearn-0\\\\\\\"\\\\n transport: \\\\\\\"rest\\\\\\\"\\\\n predictors:\\\\n - annotations:\\\\n seldonio/no-engine: \\\\\\\"true\\\\\\\"\\\\n componentSpecs:\\\\n - spec:\\\\n containers:\\\\n - name: classifier\\\\n env:\\\\n - name: GUNICORN_THREADS\\\\n value: \\\\\\\"1\\\\\\\"\\\\n - name: GUNICORN_WORKERS\\\\n value: \\\\\\\"5\\\\\\\"\\\\n graph:\\\\n children: []\\\\n implementation: SKLEARN_SERVER\\\\n modelUri: gs://seldon-models/sklearn/iris\\\\n name: classifier\\\\n name: default\\\\n replicas: 1\\\\n\\\"}}\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.289Z\" level=info msg=\"Loading manifest to /tmp/manifest.yaml\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.289Z\" level=info msg=\"kubectl create -f /tmp/manifest.yaml -o json\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.808Z\" level=info msg=argo/SeldonDeployment.machinelearning.seldon.io/sklearn-0\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.808Z\" level=info msg=\"Starting SIGUSR2 signal monitor\"\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-2323867814: time=\"2021-06-28T17:27:52.808Z\" level=info msg=\"No output parameters\"\u001b[0m\n", + "\u001b[32mseldon-benchmark-process-2388065488: Waiting for deployment \"sklearn-0-default-0-classifier\" rollout to finish: 0 of 1 updated replicas are available...\u001b[0m\n", + "\u001b[32mseldon-benchmark-process-2388065488: deployment \"sklearn-0-default-0-classifier\" successfully rolled out\u001b[0m\n", + "\u001b[31mseldon-benchmark-process-3406592537: {\"latencies\":{\"total\":29976397800,\"mean\":4636720,\"50th\":4085862,\"90th\":6830905,\"95th\":7974040,\"99th\":12617402,\"max\":37090400,\"min\":2523500},\"bytes_in\":{\"total\":1183095,\"mean\":183},\"bytes_out\":{\"total\":219810,\"mean\":34},\"earliest\":\"2021-06-28T17:28:30.4200499Z\",\"latest\":\"2021-06-28T17:29:00.4217614Z\",\"end\":\"2021-06-28T17:29:00.4249416Z\",\"duration\":30001711500,\"wait\":3180200,\"requests\":6465,\"rate\":215.48770642634838,\"throughput\":215.46486701700042,\"success\":1,\"status_codes\":{\"200\":6465},\"errors\":[],\"params\":{\"name\":\"sklearn-0\",\"replicas\":\"1\",\"serverWorkers\":\"5\",\"serverThreads\":\"1\",\"modelUri\":\"gs://seldon-models/sklearn/iris\",\"image\":\"\",\"server\":\"SKLEARN_SERVER\",\"apiType\":\"rest\",\"requestsCpu\":\"2000Mi\",\"requestsMemory\":\"100Mi\",\"limitsCpu\":\"2000Mi\",\"limitsMemory\":\"1000Mi\",\"benchmarkCpu\":\"2\",\"concurrency\":\"1\",\"duration\":\"30s\",\"rate\":\"0\",\"disableOrchestrator\":\"true\"}}\u001b[0m\n", + "\u001b[37mseldon-benchmark-process-1122797932: seldondeployment.machinelearning.seldon.io \"sklearn-0\" deleted\u001b[0m\n" + ] + } + ], + "source": [ + "!argo logs -f seldon-benchmark-process -n argo" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Name: seldon-benchmark-process\r\n", + "Namespace: argo\r\n", + "ServiceAccount: default\r\n", + "Status: Running\r\n", + "Conditions: \r\n", + " PodRunning False\r\n", + "Created: Mon Jun 28 18:38:12 +0100 (4 minutes ago)\r\n", + "Started: Mon Jun 28 18:38:12 +0100 (4 minutes ago)\r\n", + "Duration: 4 minutes 53 seconds\r\n", + "Progress: 14/15\r\n", + "ResourcesDuration: 4m41s*(1 cpu),4m41s*(100Mi memory)\r\n", + "\r\n", + "\u001b[39mSTEP\u001b[0m TEMPLATE PODNAME DURATION MESSAGE\r\n", + " \u001b[36m●\u001b[0m seldon-benchmark-process seldon-benchmark-process \r\n", + " ├───\u001b[32m✔\u001b[0m generate-parameters generate-parameters-template seldon-benchmark-process-635956972 3s \r\n", + " └─┬─\u001b[32m✔\u001b[0m run-benchmark-iteration(0:apiType:rest,benchmarkCpu:2,concurrency:1,disableOrchestrator:true,duration:30s,image:,limitsCpu:2000Mi,limitsMemory:1000Mi,modelUri:gs://seldon-models/sklearn/iris,name:sklearn-0,params:{\\\"name\\\": \\\"sklearn-0\\\", \\\"replicas\\\": \\\"1\\\", \\\"serverWorkers\\\": \\\"5\\\", \\\"serverThreads\\\": \\\"1\\\", \\\"modelUri\\\": \\\"gs://seldon-models/sklearn/iris\\\", \\\"image\\\": \\\"\\\", \\\"server\\\": \\\"SKLEARN_SERVER\\\", \\\"apiType\\\": \\\"rest\\\", \\\"requestsCpu\\\": \\\"2000Mi\\\", \\\"requestsMemory\\\": \\\"100Mi\\\", \\\"limitsCpu\\\": \\\"2000Mi\\\", \\\"limitsMemory\\\": \\\"1000Mi\\\", \\\"benchmarkCpu\\\": \\\"2\\\", \\\"concurrency\\\": \\\"1\\\", \\\"duration\\\": \\\"30s\\\", \\\"rate\\\": \\\"0\\\", \\\"disableOrchestrator\\\": \\\"true\\\"},rate:0,replicas:1,requestsCpu:2000Mi,requestsMemory:100Mi,server:SKLEARN_SERVER,serverThreads:1,serverWorkers:5) run-benchmark-iteration-step-template \r\n", + " │ ├───\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-benchmark-process-2323867814 1s \r\n", + " │ ├───\u001b[32m✔\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-benchmark-process-2388065488 15s \r\n", + " │ ├─┬─\u001b[39m○\u001b[0m run-benchmark-grpc run-benchmark-template-grpc when 'rest == grpc' evaluated false \r\n", + " │ │ └─\u001b[32m✔\u001b[0m run-benchmark-rest run-benchmark-template-rest seldon-benchmark-process-3406592537 31s \r\n", + " │ └───\u001b[32m✔\u001b[0m delete-seldon-resource delete-seldon-resource-template seldon-benchmark-process-1122797932 3s \r\n", + " ├─\u001b[32m✔\u001b[0m run-benchmark-iteration(1:apiType:rest,benchmarkCpu:2,concurrency:1,disableOrchestrator:false,duration:30s,image:,limitsCpu:2000Mi,limitsMemory:1000Mi,modelUri:gs://seldon-models/sklearn/iris,name:sklearn-1,params:{\\\"name\\\": \\\"sklearn-1\\\", \\\"replicas\\\": \\\"1\\\", \\\"serverWorkers\\\": \\\"5\\\", \\\"serverThreads\\\": \\\"1\\\", \\\"modelUri\\\": \\\"gs://seldon-models/sklearn/iris\\\", \\\"image\\\": \\\"\\\", \\\"server\\\": \\\"SKLEARN_SERVER\\\", \\\"apiType\\\": \\\"rest\\\", \\\"requestsCpu\\\": \\\"2000Mi\\\", \\\"requestsMemory\\\": \\\"100Mi\\\", \\\"limitsCpu\\\": \\\"2000Mi\\\", \\\"limitsMemory\\\": \\\"1000Mi\\\", \\\"benchmarkCpu\\\": \\\"2\\\", \\\"concurrency\\\": \\\"1\\\", \\\"duration\\\": \\\"30s\\\", \\\"rate\\\": \\\"0\\\", \\\"disableOrchestrator\\\": \\\"false\\\"},rate:0,replicas:1,requestsCpu:2000Mi,requestsMemory:100Mi,server:SKLEARN_SERVER,serverThreads:1,serverWorkers:5) run-benchmark-iteration-step-template \r\n", + " │ ├───\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-benchmark-process-1282498819 2s \r\n", + " │ ├───\u001b[32m✔\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-benchmark-process-634593 17s \r\n", + " │ ├─┬─\u001b[39m○\u001b[0m run-benchmark-grpc run-benchmark-template-grpc when 'rest == grpc' evaluated false \r\n", + " │ │ └─\u001b[32m✔\u001b[0m run-benchmark-rest run-benchmark-template-rest seldon-benchmark-process-692934928 32s \r\n", + " │ └───\u001b[32m✔\u001b[0m delete-seldon-resource delete-seldon-resource-template seldon-benchmark-process-3935684561 2s \r\n", + " ├─\u001b[32m✔\u001b[0m run-benchmark-iteration(2:apiType:grpc,benchmarkCpu:2,concurrency:1,disableOrchestrator:true,duration:30s,image:,limitsCpu:2000Mi,limitsMemory:1000Mi,modelUri:gs://seldon-models/sklearn/iris,name:sklearn-2,params:{\\\"name\\\": \\\"sklearn-2\\\", \\\"replicas\\\": \\\"1\\\", \\\"serverWorkers\\\": \\\"5\\\", \\\"serverThreads\\\": \\\"1\\\", \\\"modelUri\\\": \\\"gs://seldon-models/sklearn/iris\\\", \\\"image\\\": \\\"\\\", \\\"server\\\": \\\"SKLEARN_SERVER\\\", \\\"apiType\\\": \\\"grpc\\\", \\\"requestsCpu\\\": \\\"2000Mi\\\", \\\"requestsMemory\\\": \\\"100Mi\\\", \\\"limitsCpu\\\": \\\"2000Mi\\\", \\\"limitsMemory\\\": \\\"1000Mi\\\", \\\"benchmarkCpu\\\": \\\"2\\\", \\\"concurrency\\\": \\\"1\\\", \\\"duration\\\": \\\"30s\\\", \\\"rate\\\": \\\"0\\\", \\\"disableOrchestrator\\\": \\\"true\\\"},rate:0,replicas:1,requestsCpu:2000Mi,requestsMemory:100Mi,server:SKLEARN_SERVER,serverThreads:1,serverWorkers:5) run-benchmark-iteration-step-template \r\n", + " │ ├───\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-benchmark-process-637309828 1s \r\n", + " │ ├───\u001b[32m✔\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-benchmark-process-2284480586 19s \r\n", + " │ ├─┬─\u001b[32m✔\u001b[0m run-benchmark-grpc run-benchmark-template-grpc seldon-benchmark-process-2580139445 32s \r\n", + " │ │ └─\u001b[39m○\u001b[0m run-benchmark-rest run-benchmark-template-rest when 'grpc == rest' evaluated false \r\n", + " │ └───\u001b[32m✔\u001b[0m delete-seldon-resource delete-seldon-resource-template seldon-benchmark-process-757645342 3s \r\n", + " └─\u001b[36m●\u001b[0m run-benchmark-iteration(3:apiType:grpc,benchmarkCpu:2,concurrency:1,disableOrchestrator:false,duration:30s,image:,limitsCpu:2000Mi,limitsMemory:1000Mi,modelUri:gs://seldon-models/sklearn/iris,name:sklearn-3,params:{\\\"name\\\": \\\"sklearn-3\\\", \\\"replicas\\\": \\\"1\\\", \\\"serverWorkers\\\": \\\"5\\\", \\\"serverThreads\\\": \\\"1\\\", \\\"modelUri\\\": \\\"gs://seldon-models/sklearn/iris\\\", \\\"image\\\": \\\"\\\", \\\"server\\\": \\\"SKLEARN_SERVER\\\", \\\"apiType\\\": \\\"grpc\\\", \\\"requestsCpu\\\": \\\"2000Mi\\\", \\\"requestsMemory\\\": \\\"100Mi\\\", \\\"limitsCpu\\\": \\\"2000Mi\\\", \\\"limitsMemory\\\": \\\"1000Mi\\\", \\\"benchmarkCpu\\\": \\\"2\\\", \\\"concurrency\\\": \\\"1\\\", \\\"duration\\\": \\\"30s\\\", \\\"rate\\\": \\\"0\\\", \\\"disableOrchestrator\\\": \\\"false\\\"},rate:0,replicas:1,requestsCpu:2000Mi,requestsMemory:100Mi,server:SKLEARN_SERVER,serverThreads:1,serverWorkers:5) run-benchmark-iteration-step-template \r\n", + " ├───\u001b[32m✔\u001b[0m create-seldon-resource create-seldon-resource-template seldon-benchmark-process-1376808213 1s \r\n", + " └───\u001b[33m◷\u001b[0m wait-seldon-resource wait-seldon-resource-template seldon-benchmark-process-3668579423 7s \r\n" + ] + } + ], + "source": [ + "!argo get seldon-benchmark-process -n argo" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Process the results\n", + "\n", + "We can now print the results in a consumable format." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Deeper Analysis\n", + "Now that we have all the parameters, we can do a deeper analysis" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append(\"../../../testing/scripts\")\n", + "from seldon_e2e_utils import bench_results_from_output_logs\n", + "import pandas as pd\n", + "\n", + "results = bench_results_from_output_logs(\"seldon-benchmark-process\", namespace=\"argo\")\n", + "df = pd.DataFrame.from_dict(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
mean50th90th95th99ththroughputAchievedsuccesserrorsnamereplicas...apiTyperequestsCpurequestsMemorylimitsCpulimitsMemorybenchmarkCpuconcurrencydurationratedisableOrchestrator
04.5733024.0186356.2257107.48087813.893386218.51881165590sklearn-01...rest2000Mi100Mi2000Mi1000Mi2130s0true
14.5651453.9390326.7853937.92870413.315820218.89280665680sklearn-11...rest2000Mi100Mi2000Mi1000Mi2130s0false
23.7473193.2123005.6516006.8587009.191800258.59574677571sklearn-21...grpc2000Mi100Mi2000Mi1000Mi2130s0true
34.2718793.8558006.4958007.3535008.980500226.93006368071sklearn-31...grpc2000Mi100Mi2000Mi1000Mi2130s0false
\n", + "

4 rows × 25 columns

\n", + "
" + ], + "text/plain": [ + " mean 50th 90th 95th 99th throughputAchieved \\\n", + "0 4.573302 4.018635 6.225710 7.480878 13.893386 218.518811 \n", + "1 4.565145 3.939032 6.785393 7.928704 13.315820 218.892806 \n", + "2 3.747319 3.212300 5.651600 6.858700 9.191800 258.595746 \n", + "3 4.271879 3.855800 6.495800 7.353500 8.980500 226.930063 \n", + "\n", + " success errors name replicas ... apiType requestsCpu \\\n", + "0 6559 0 sklearn-0 1 ... rest 2000Mi \n", + "1 6568 0 sklearn-1 1 ... rest 2000Mi \n", + "2 7757 1 sklearn-2 1 ... grpc 2000Mi \n", + "3 6807 1 sklearn-3 1 ... grpc 2000Mi \n", + "\n", + " requestsMemory limitsCpu limitsMemory benchmarkCpu concurrency duration \\\n", + "0 100Mi 2000Mi 1000Mi 2 1 30s \n", + "1 100Mi 2000Mi 1000Mi 2 1 30s \n", + "2 100Mi 2000Mi 1000Mi 2 1 30s \n", + "3 100Mi 2000Mi 1000Mi 2 1 30s \n", + "\n", + " rate disableOrchestrator \n", + "0 0 true \n", + "1 0 false \n", + "2 0 true \n", + "3 0 false \n", + "\n", + "[4 rows x 25 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Workflow 'seldon-benchmark-process' deleted\r\n" + ] + } + ], + "source": [ + "!argo delete seldon-benchmark-process -n argo || echo \"Argo workflow already deleted or not exists\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/testing/benchmarking/ghz/Makefile b/testing/benchmarking/ghz/Makefile index cd52801dc5..a955c385f4 100644 --- a/testing/benchmarking/ghz/Makefile +++ b/testing/benchmarking/ghz/Makefile @@ -1,4 +1,4 @@ -VERSION=v0.55.0 +VERSION=v0.95.0 IMG=seldonio/ghz:${VERSION} docker-build: diff --git a/testing/resources/argo-configmap.yaml b/testing/resources/argo-configmap.yaml new file mode 100644 index 0000000000..0530c60b42 --- /dev/null +++ b/testing/resources/argo-configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-controller-configmap +data: + containerRuntimeExecutor: k8sapi diff --git a/testing/scripts/Makefile b/testing/scripts/Makefile index 4d9c6f33aa..2689282f40 100644 --- a/testing/scripts/Makefile +++ b/testing/scripts/Makefile @@ -181,6 +181,13 @@ test_notebooks: build_protos install -m "notebooks" \ -W ignore test_notebooks.py::TestNotebooks 2>&1 +test_benchmark: build_protos install + pytest \ + --verbose \ + -s \ + -W ignore \ + -m "benchmark" 2>&1 + .PHONY: clean clean: rm -f proto/prediction* diff --git a/testing/scripts/conftest.py b/testing/scripts/conftest.py index 91b1ccce73..819160aa04 100644 --- a/testing/scripts/conftest.py +++ b/testing/scripts/conftest.py @@ -14,24 +14,6 @@ def _to_python_bool(val): return val.lower() in {"1", "t", "true"} -SELDON_E2E_TESTS_POD_INFORMATION = _to_python_bool( - os.getenv("SELDON_E2E_TESTS_POD_INFORMATION", default="false") -) - - -@pytest.fixture(scope="session", autouse=SELDON_E2E_TESTS_POD_INFORMATION) -def run_pod_information_in_background(request): - # This command runs the pod information and prints it in the background - # every time there's a new update - run( - ( - "kubectl get pods --all-namespaces -w | " - + 'awk \'{print "\\nPODS UPDATE: "$0"\\n"}\' & ' - ), - shell=True, - ) - - @pytest.fixture(scope="module") def s2i_python_version(): """Return version of s2i images, the IMAGE_VERSION, e.g. 0.17-SNAPSHOT.""" @@ -59,6 +41,41 @@ def namespace(request): run(f"kubectl delete namespace {namespace}", shell=True) +def install_argo(): + kwargs = { + "check": True, + "shell": True, + } + + run("kubectl create namespace argo", **kwargs) + run( + "kubectl apply -n argo -f https://raw.githubusercontent.com/argoproj/argo/stable/manifests/install.yaml", + **kwargs, + ) + run("kubectl rollout status -n argo deployment/argo-server", **kwargs) + run("kubectl rollout status -n argo deployment/workflow-controller", **kwargs) + run( + "kubectl create rolebinding argo-default-admin --clusterrole=admin --serviceaccount=argo:default -n argo", + **kwargs, + ) + run( + "kubectl create rolebinding argo-seldon-workflow --clusterrole=seldon-manager-role-seldon-system --serviceaccount=argo:default -n argo", + **kwargs, + ) + run("kubectl apply -n argo -f ../resources/argo-configmap.yaml", **kwargs) + + +def delete_argo(): + run("kubectl delete namespace argo", check=True, shell=True) + + +@pytest.fixture() +def argo_worfklows(scope="module"): + install_argo() + yield + delete_argo() + + @pytest.fixture def seldon_version(request): """ diff --git a/testing/scripts/dev_requirements.txt b/testing/scripts/dev_requirements.txt index 7044c87d98..0152648ea5 100644 --- a/testing/scripts/dev_requirements.txt +++ b/testing/scripts/dev_requirements.txt @@ -12,6 +12,7 @@ alibi==0.5.2 shap==0.35.0 matplotlib==3.1.3 tensorflow==2.4.0 +tabulate==0.8.7 # 2nd lvl dep on cov required to avoid sqllite dep coverage==4.5.4 diff --git a/testing/scripts/kind_test_all.sh b/testing/scripts/kind_test_all.sh index a5d0202356..9e45510a16 100755 --- a/testing/scripts/kind_test_all.sh +++ b/testing/scripts/kind_test_all.sh @@ -103,12 +103,18 @@ if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then echo "SKIPPING EXECUTOR IMAGE BUILD..." fi - echo "Build test models" - make kind_build_test_models - KIND_BUILD_EXIT_VALUE=$? - if [[ $KIND_BUILD_EXIT_VALUE -gt 0 ]]; then - echo "Kind build has errors" - return 1 + echo "Files changed in test models folder:" + git --no-pager diff --exit-code --name-only origin/master ../../examples/models/mean_classifier/ ../../testing/docker/echo-model/ ../../testing/docker/fixed-model + TEST_MODELS_MODIFIED=$? + if [[ $TEST_MODELS_MODIFIED -gt 0 ]]; then + make kind_build_test_models + KIND_BUILD_EXIT_VALUE=$? + if [[ $KIND_BUILD_EXIT_VALUE -gt 0 ]]; then + echo "Kind build has errors" + return 1 + fi + else + echo "SKIPPING TEST MODEL BUILD..." fi echo "Files changed in prepackaged, python, or wrapper folder:" @@ -206,6 +212,8 @@ if [[ ${KIND_EXIT_VALUE} -eq 0 ]]; then make test_parallel test_sequential elif [ "$TESTS_TO_RUN" == "parallel" ]; then make test_parallel + elif [ "$TESTS_TO_RUN" == "benchmark" ]; then + make test_benchmark fi TEST_EXIT_VALUE=$? if [[ $TEST_EXIT_VALUE -gt 0 ]]; then diff --git a/testing/scripts/seldon_e2e_utils.py b/testing/scripts/seldon_e2e_utils.py index 3df7f956c1..fe5dec9eb8 100644 --- a/testing/scripts/seldon_e2e_utils.py +++ b/testing/scripts/seldon_e2e_utils.py @@ -5,10 +5,11 @@ import subprocess import time from concurrent.futures import ThreadPoolExecutor, wait -from subprocess import Popen, run +from subprocess import CalledProcessError, Popen, run import grpc import numpy as np +import pandas as pd import requests from google.protobuf import empty_pb2 from requests.auth import HTTPBasicAuth @@ -22,6 +23,8 @@ TESTING_ROOT_PATH = os.path.dirname(os.path.dirname(__file__)) RESOURCES_PATH = os.path.join(TESTING_ROOT_PATH, "resources") +BENCHMARK_PARALLELISM = 2 + def get_seldon_version(): ret = Popen("cat ../../version.txt", shell=True, stdout=subprocess.PIPE) @@ -628,3 +631,202 @@ def assert_model(sdep_name, namespace, initial=False, endpoint=API_AMBASSADOR): def to_resources_path(file_name): return os.path.join(RESOURCES_PATH, file_name) + + +def post_comment_in_pr(body, check=False): + try: + run(f'jx gitops pr comment --comment "{body}"', shell=True, check=True) + except Exception: + logging.exception("Error posting comment with results") + if check: + raise + + +def create_and_run_script(folder, notebook): + run( + f"jupyter nbconvert --template ../../notebooks/convert.tpl --to script {folder}/{notebook}.ipynb", + shell=True, + check=True, + ) + run(f"chmod u+x {folder}/{notebook}.py", shell=True, check=True) + try: + run( + f"cd {folder} && ./{notebook}.py", + shell=True, + check=True, + encoding="utf-8", + ) + except CalledProcessError as e: + logging.error( + f"failed notebook test {notebook} stdout:{e.stdout}, stderr:{e.stderr}" + ) + run("kubectl delete sdep --all", shell=True, check=False) + raise e + + +def bench_results_from_output_logs(name, namespace="argo", print_results=True): + + output = run( + f"argo logs --no-color {name} -n {namespace}", + stdout=subprocess.PIPE, + encoding="utf-8", + check=True, + shell=True, + ) + + output.check_returncode() + + log_array = output.stdout.split("\n") + + results = [] + for log in log_array: + if "latenc" in log: + # Only process if contains results of benchmark + log_clean = json.loads(":".join(log.split(":")[1:])) + result = parse_bench_results_from_log( + log_clean, print_results=print_results + ) + results.append(result) + + return results + + +def print_benchmark_results(final): + print("-----") + print("ParamNames:", final["params"].keys()) + print("ParamNames:", final["params"].values()) + print("\tLatencies:") + print("\t\tmean:", final["mean"], "ms") + print("\t\t50th:", final["50th"], "ms") + print("\t\t90th:", final["90th"], "ms") + print("\t\t95th:", final["95th"], "ms") + print("\t\t99th:", final["99th"], "ms") + print("") + print("\tRate:", str(final["throughputAchieved"]) + "/s") + print("\tSuccess:", final["success"]) + print("\tErrors:", final["errors"]) + + +def parse_bench_results_from_log( + results_log, + print_results=True, +): + final = {} + # For GHZ / grpc + if "average" in results_log: + final["mean"] = results_log["average"] / 1e6 + if results_log.get("latencyDistribution", False): + final["50th"] = results_log["latencyDistribution"][-5]["latency"] / 1e6 + final["90th"] = results_log["latencyDistribution"][-3]["latency"] / 1e6 + final["95th"] = results_log["latencyDistribution"][-2]["latency"] / 1e6 + final["99th"] = results_log["latencyDistribution"][-1]["latency"] / 1e6 + final["throughputAchieved"] = results_log["rps"] + final["success"] = results_log["statusCodeDistribution"].get("OK", 0) + final["errors"] = ( + sum(results_log["statusCodeDistribution"].values()) - final["success"] + ) + # For vegeta / rest + else: + final["mean"] = results_log["latencies"]["mean"] / 1e6 + final["50th"] = results_log["latencies"]["50th"] / 1e6 + final["90th"] = results_log["latencies"]["90th"] / 1e6 + final["95th"] = results_log["latencies"]["95th"] / 1e6 + final["99th"] = results_log["latencies"]["99th"] / 1e6 + final["throughputAchieved"] = results_log["throughput"] + final["success"] = results_log["status_codes"].get("200", 0) + final["errors"] = sum(results_log["status_codes"].values()) - final["success"] + # Including parameters + for k in results_log["params"].keys(): + final[k] = results_log["params"][k] + return final + + +def run_benchmark_and_capture_results( + name="seldon-batch-job", + namespace="argo", + parallelism=BENCHMARK_PARALLELISM, + replicas_list=["1"], + server_workers_list=["5"], + server_threads_list=["1"], + model_uri_list=["gs://seldon-models/sklearn/iris"], + server_list=["SKLEARN_SERVER"], + api_type_list=["rest"], + requests_cpu_list=["2000Mi"], + requests_memory_list=["500Mi"], + limits_cpu_list=["2000Mi"], + limits_memory_list=["500Mi"], + disable_orchestrator_list=["false"], + benchmark_cpu_list=["1"], + benchmark_concurrency_list=["1"], + benchmark_duration_list=["30s"], + benchmark_rate_list=["0"], + benchmark_data={"data": {"ndarray": [[1, 2, 3, 4]]}}, +): + + data_str = ( + json.dumps(benchmark_data) + .replace("{", "\\{") + .replace("}", "\\}") + .replace(",", "\\,") + ) + + delim = "|" + + replicas = delim.join(replicas_list) + server_workers = delim.join(server_workers_list) + server_threads = delim.join(server_threads_list) + model_uri = delim.join(model_uri_list) + server = delim.join(server_list) + api_type = delim.join(api_type_list) + requests_cpu = delim.join(requests_cpu_list) + requests_memory = delim.join(requests_memory_list) + limits_cpu = delim.join(limits_cpu_list) + limits_memory = delim.join(limits_memory_list) + disable_orchestrator = delim.join(disable_orchestrator_list) + benchmark_cpu = delim.join(benchmark_cpu_list) + benchmark_concurrency = delim.join(benchmark_concurrency_list) + benchmark_duration = delim.join(benchmark_duration_list) + benchmark_rate = delim.join(benchmark_rate_list) + + kwargs = { + "shell": True, + "check": True, + } + + run( + f""" + helm template seldon-benchmark-workflow ../../helm-charts/seldon-benchmark-workflow/ \\ + --set workflow.namespace="{namespace}" \\ + --set workflow.name="{name}" \\ + --set workflow.parallelism="{parallelism}" \\ + --set seldonDeployment.name="{name}-sdep" \\ + --set seldonDeployment.replicas="{replicas}" \\ + --set seldonDeployment.serverWorkers="{server_workers}" \\ + --set seldonDeployment.serverThreads="{server_threads}" \\ + --set seldonDeployment.modelUri="{model_uri}" \\ + --set seldonDeployment.server="{server}" \\ + --set seldonDeployment.apiType="{api_type}" \\ + --set seldonDeployment.requests.cpu="{requests_cpu}" \\ + --set seldonDeployment.requests.memory="{requests_memory}" \\ + --set seldonDeployment.limits.cpu="{limits_cpu}" \\ + --set seldonDeployment.limits.memory="{limits_memory}" \\ + --set seldonDeployment.disableOrchestrator="{disable_orchestrator}" \\ + --set benchmark.cpu="{benchmark_cpu}" \\ + --set benchmark.concurrency="{benchmark_concurrency}" \\ + --set benchmark.duration="{benchmark_duration}" \\ + --set benchmark.rate="{benchmark_rate}" \\ + --set benchmark.data='{data_str}' \\ + | argo submit - + """, + **kwargs, + ) + run("argo list -n {namespace}", **kwargs) + run(f"argo logs -n {namespace} -f {name}", **kwargs) + + results = bench_results_from_output_logs(name) + + df_results = pd.DataFrame.from_dict(results) + + run(f"argo delete -n {namespace} {name}", **kwargs) + + return df_results diff --git a/testing/scripts/test_benchmark.py b/testing/scripts/test_benchmark.py new file mode 100644 index 0000000000..0b3e6b1996 --- /dev/null +++ b/testing/scripts/test_benchmark.py @@ -0,0 +1,45 @@ +import json +import logging + +import pandas as pd +import pytest + +from seldon_e2e_utils import post_comment_in_pr, run_benchmark_and_capture_results + + +@pytest.mark.benchmark +@pytest.mark.usefixtures("argo_worfklows") +def test_service_orchestrator(): + + df = run_benchmark_and_capture_results( + api_type_list=["rest", "grpc"], + disable_orchestrator_list=["false", "true"], + ) + + result_body = "# Benchmark results\n\n" + result_body += str(df.to_markdown()) + post_comment_in_pr(result_body) + + # Ensure all mean performance latency below 7 ms + assert all(df["mean"] < 10) + # Ensure 99th percentiles are not spiking above 15ms + assert all(df["mean"] < 15) + # Ensure throughput is above 200 rps for REST + assert all(df[df["apiType"] == "rest"]["throughputAchieved"] > 140) + # Ensure throughput is above 250 rps for GRPC + assert all(df[df["apiType"] == "grpc"]["throughputAchieved"] > 160) + # Validate latenc added by adding service orchestrator is lower than 4ms + assert all( + ( + df[df["disableOrchestrator"] == "true"]["mean"].values + - df[df["disableOrchestrator"] == "false"]["mean"].values + ) + < 2 + ) + assert all( + ( + df[df["disableOrchestrator"] == "true"]["99th"].values + - df[df["disableOrchestrator"] == "false"]["99th"].values + ) + < 2 + ) diff --git a/testing/scripts/test_notebooks.py b/testing/scripts/test_notebooks.py index d47291d102..6e221c236e 100644 --- a/testing/scripts/test_notebooks.py +++ b/testing/scripts/test_notebooks.py @@ -1,31 +1,6 @@ -import logging -from subprocess import PIPE, CalledProcessError, run - import pytest - -def create_and_run_script(folder, notebook): - run( - f"jupyter nbconvert --template ../../notebooks/convert.tpl --to script {folder}/{notebook}.ipynb", - shell=True, - check=True, - ) - run(f"chmod u+x {folder}/{notebook}.py", shell=True, check=True) - try: - run( - f"cd {folder} && ./{notebook}.py", - shell=True, - check=True, - stdout=PIPE, - stderr=PIPE, - encoding="utf-8", - ) - except CalledProcessError as e: - logging.error( - f"failed notebook test {notebook} stdout:{e.stdout}, stderr:{e.stderr}" - ) - run("kubectl delete sdep --all", shell=True, check=False) - raise e +from seldon_e2e_utils import create_and_run_script @pytest.mark.flaky(max_runs=2)