diff --git a/README.md b/README.md index 0ac91f30..33389614 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,8 @@ Tempo provides a unified interface to multiple MLOps projects that enable data s * Deploy locally to Docker to test with Docker runtimes. * Deploy to production on Kubernetes * Extract declarative Kubernetes yaml to follow GitOps workflows. - * Supporting a wide range of production runtimes + * Supporting Seldon production runtimes * Seldon Core open source - * KFServing open source * Seldon Deploy enterprise * Create stateful services. Examples: * Multi-Armed Bandits. diff --git a/docs/examples/asyncio/README.ipynb b/docs/examples/asyncio/README.ipynb index 8b7f5c97..44d76efc 100644 --- a/docs/examples/asyncio/README.ipynb +++ b/docs/examples/asyncio/README.ipynb @@ -40,31 +40,12 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "alert-surge", "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34m.\u001b[00m\r\n", - "├── \u001b[01;34martifacts\u001b[00m\r\n", - "│   ├── \u001b[01;34mclassifier\u001b[00m\r\n", - "│   ├── \u001b[01;34msklearn\u001b[00m\r\n", - "│   └── \u001b[01;34mxgboost\u001b[00m\r\n", - "└── \u001b[01;34msrc\u001b[00m\r\n", - " ├── constants.py\r\n", - " ├── data.py\r\n", - " ├── tempo.py\r\n", - " └── train.py\r\n", - "\r\n", - "5 directories, 4 files\r\n" - ] - } - ], + "outputs": [], "source": [ "!tree -P \"*.py\" -I \"__init__.py|__pycache__\" -L 2" ] @@ -84,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "equipped-silence", "metadata": {}, "outputs": [], @@ -98,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "reported-hurricane", "metadata": { "code_folding": [] @@ -134,26 +115,10 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "lesser-reply", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[18:05:52] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/clive/anaconda3/envs/tempo-examples/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n", - " warnings.warn(label_encoder_deprecation_msg, UserWarning)\n" - ] - } - ], + "outputs": [], "source": [ "from src.data import IrisData\n", "from src.train import train_sklearn, train_xgboost\n", @@ -177,7 +142,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "smoking-tribe", "metadata": {}, "outputs": [], @@ -187,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "regular-balance", "metadata": { "code_folding": [] @@ -243,38 +208,20 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "considered-terminology", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "artifacts/classifier/conda.yaml\r\n" - ] - } - ], + "outputs": [], "source": [ "!ls artifacts/classifier/conda.yaml" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "rural-mathematics", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-330c15d8-a189-45a6-abc3-a27f39b6a7c5' to '/home/clive/work/mlops/fork-tempo/docs/examples/asyncio/artifacts/classifier/environment.tar.gz'\n", - "[########################################] | 100% Completed | 11.2s\n" - ] - } - ], + "outputs": [], "source": [ "import tempo\n", "\n", @@ -293,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "56037cc4", "metadata": {}, "outputs": [], @@ -304,21 +251,10 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "cb489575", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "array([2.], dtype=float32)" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "import numpy as np\n", "await remote_model.predict(np.array([[1, 2, 3, 4]]))" @@ -326,19 +262,10 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "dc6c0829", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[1.]\n", - "[[0.97329617 0.02412145 0.00258233]]\n" - ] - } - ], + "outputs": [], "source": [ "print(await remote_model.predict(np.array([[0, 0, 0,0]])))\n", "print(await remote_model.predict(np.array([[5.964,4.006,2.081,1.031]])))" @@ -346,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "id": "6fa56a63", "metadata": {}, "outputs": [], diff --git a/docs/examples/asyncio/README.md b/docs/examples/asyncio/README.md index 93fb6b66..f20362a0 100644 --- a/docs/examples/asyncio/README.md +++ b/docs/examples/asyncio/README.md @@ -23,20 +23,6 @@ conda env create --name tempo-examples --file conda/tempo-examples.yaml !tree -P "*.py" -I "__init__.py|__pycache__" -L 2 ``` - . - ├── artifacts - │   ├── classifier - │   ├── sklearn - │   └── xgboost - └── src - ├── constants.py - ├── data.py - ├── tempo.py - └── train.py - - 5 directories, 4 files - - ## Train Models This section is where as a data scientist you do your work of training models and creating artfacts. @@ -94,13 +80,6 @@ train_sklearn(data) train_xgboost(data) ``` - [18:05:52] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior. - - - /home/clive/anaconda3/envs/tempo-examples/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1]. - warnings.warn(label_encoder_deprecation_msg, UserWarning) - - ## Create Tempo Artifacts Here we create the Tempo models and orchestration Pipeline for our final service using our models. @@ -160,9 +139,6 @@ In preparation for running our models we save the Python environment needed for !ls artifacts/classifier/conda.yaml ``` - artifacts/classifier/conda.yaml - - ```python import tempo @@ -170,11 +146,6 @@ import tempo tempo.save(classifier) ``` - Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-330c15d8-a189-45a6-abc3-a27f39b6a7c5' to '/home/clive/work/mlops/fork-tempo/docs/examples/asyncio/artifacts/classifier/environment.tar.gz' - [########################################] | 100% Completed | 11.2s - - ## Test Locally on Docker Here we test our models using production images but running locally on Docker. This allows us to ensure the final production deployed model will behave as expected when deployed. @@ -192,22 +163,11 @@ await remote_model.predict(np.array([[1, 2, 3, 4]])) ``` - - - array([2.], dtype=float32) - - - - ```python print(await remote_model.predict(np.array([[0, 0, 0,0]]))) print(await remote_model.predict(np.array([[5.964,4.006,2.081,1.031]]))) ``` - [1.] - [[0.97329617 0.02412145 0.00258233]] - - ```python remote_model.undeploy() diff --git a/docs/examples/control-environments/pipeline-as-class.md b/docs/examples/control-environments/pipeline-as-class.md index cedcc842..c38a5fe9 100644 --- a/docs/examples/control-environments/pipeline-as-class.md +++ b/docs/examples/control-environments/pipeline-as-class.md @@ -116,7 +116,7 @@ dependencies: ```python from tempo.serve.metadata import ModelFramework, KubernetesOptions -from tempo.kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol from tempo.seldon.k8s import SeldonKubernetesRuntime from tempo.seldon.docker import SeldonDockerRuntime diff --git a/docs/examples/control-environments/pipeline-function.md b/docs/examples/control-environments/pipeline-function.md index 5e04e00c..3e2ec8e6 100644 --- a/docs/examples/control-environments/pipeline-function.md +++ b/docs/examples/control-environments/pipeline-function.md @@ -114,7 +114,7 @@ dependencies: ```python from tempo.serve.metadata import ModelFramework, KubernetesOptions -from tempo.kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol from tempo.seldon.k8s import SeldonKubernetesRuntime from tempo.seldon.docker import SeldonDockerRuntime diff --git a/docs/examples/custom-model/README.ipynb b/docs/examples/custom-model/README.ipynb index 34213478..c7868343 100644 --- a/docs/examples/custom-model/README.ipynb +++ b/docs/examples/custom-model/README.ipynb @@ -135,7 +135,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "sample: 100%|██████████| 3000/3000 [00:05<00:00, 547.59it/s, 3 steps of size 7.77e-01. acc. prob=0.91]\n" + "sample: 100%|██████████| 3000/3000 [00:06<00:00, 445.23it/s, 3 steps of size 7.77e-01. acc. prob=0.91]\n" ] }, { @@ -335,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -352,7 +352,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -360,8 +360,8 @@ "output_type": "stream", "text": [ "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-792d47bc-3391-4a9b-949b-bc38fe1cd5dd' to '/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts/environment.tar.gz'\n", - "[########################################] | 100% Completed | 19.0s\n" + "Packing environment at '/home/clive/anaconda3/envs/tempo-7dfc12cb-53ee-44f4-ae37-fb0e9e60a4b8' to '/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts/environment.tar.gz'\n", + "[########################################] | 100% Completed | 25.2s\n" ] } ], @@ -372,7 +372,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -389,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -398,7 +398,7 @@ "array([9.673733], dtype=float32)" ] }, - "execution_count": 16, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -409,7 +409,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -431,7 +431,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -451,7 +451,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -462,7 +462,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -472,7 +472,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -487,7 +487,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -497,7 +497,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -506,7 +506,7 @@ "array([9.673733], dtype=float32)" ] }, - "execution_count": 23, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -517,7 +517,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -536,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -555,7 +555,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -571,7 +571,7 @@ " \"/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts\", \"uri\":\r\n", " \"s3://tempo/divorce\", \"platform\": \"custom\", \"inputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", " \"name\": \"marriage\"}, {\"ty\": \"numpy.ndarray\", \"name\": \"age\"}]}, \"outputs\": {\"args\":\r\n", - " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"description\": \"\"}, \"protocol\": \"tempo.kfserving.protocol.KFServingV2Protocol\",\r\n", + " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"description\": \"\"}, \"protocol\": \"tempo.protocols.v2.V2Protocol\",\r\n", " \"runtime_options\": {\"runtime\": \"tempo.seldon.SeldonKubernetesRuntime\", \"state_options\":\r\n", " {\"state_type\": \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\":\r\n", " {\"worker_endpoint\": \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\":\r\n", diff --git a/docs/examples/custom-model/README.md b/docs/examples/custom-model/README.md index 95d16ddc..f60c1e1f 100644 --- a/docs/examples/custom-model/README.md +++ b/docs/examples/custom-model/README.md @@ -103,7 +103,7 @@ from src.train import train, save, model_function mcmc = train() ``` - sample: 100%|██████████| 3000/3000 [00:05<00:00, 547.59it/s, 3 steps of size 7.77e-01. acc. prob=0.91] + sample: 100%|██████████| 3000/3000 [00:06<00:00, 445.23it/s, 3 steps of size 7.77e-01. acc. prob=0.91] @@ -250,8 +250,8 @@ save(numpyro_divorce) ``` Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-792d47bc-3391-4a9b-949b-bc38fe1cd5dd' to '/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts/environment.tar.gz' - [########################################] | 100% Completed | 19.0s + Packing environment at '/home/clive/anaconda3/envs/tempo-7dfc12cb-53ee-44f4-ae37-fb0e9e60a4b8' to '/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts/environment.tar.gz' + [########################################] | 100% Completed | 25.2s @@ -379,7 +379,7 @@ with open(os.getcwd()+"/k8s/tempo.yaml","w") as f: "/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts", "uri": "s3://tempo/divorce", "platform": "custom", "inputs": {"args": [{"ty": "numpy.ndarray", "name": "marriage"}, {"ty": "numpy.ndarray", "name": "age"}]}, "outputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "description": ""}, "protocol": "tempo.kfserving.protocol.KFServingV2Protocol", + [{"ty": "numpy.ndarray", "name": null}]}, "description": ""}, "protocol": "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": diff --git a/docs/examples/custom-model/k8s/tempo.yaml b/docs/examples/custom-model/k8s/tempo.yaml index 27e0003f..3eafbca7 100644 --- a/docs/examples/custom-model/k8s/tempo.yaml +++ b/docs/examples/custom-model/k8s/tempo.yaml @@ -7,7 +7,7 @@ metadata: "/home/clive/work/mlops/fork-tempo/docs/examples/custom-model/artifacts", "uri": "s3://tempo/divorce", "platform": "custom", "inputs": {"args": [{"ty": "numpy.ndarray", "name": "marriage"}, {"ty": "numpy.ndarray", "name": "age"}]}, "outputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "description": ""}, "protocol": "tempo.kfserving.protocol.KFServingV2Protocol", + [{"ty": "numpy.ndarray", "name": null}]}, "description": ""}, "protocol": "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": @@ -47,7 +47,7 @@ spec: true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}' - image: seldonio/mlserver:0.3.2 + image: seldonio/mlserver:0.4.1 name: numpyro-divorce graph: envSecretRefName: minio-secret diff --git a/docs/examples/explainer/README.ipynb b/docs/examples/explainer/README.ipynb index 889bc05b..d044422a 100644 --- a/docs/examples/explainer/README.ipynb +++ b/docs/examples/explainer/README.ipynb @@ -273,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 8, "id": "d8b75d35", "metadata": {}, "outputs": [ @@ -300,8 +300,8 @@ "output_type": "stream", "text": [ "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-27d7d340-70df-4095-92b7-b9ef722eda26' to '/home/clive/work/mlops/fork-tempo/docs/examples/explainer/artifacts/explainer/environment.tar.gz'\n", - "[########################################] | 100% Completed | 59.8s\n" + "Packing environment at '/home/clive/anaconda3/envs/tempo-d87b2b65-e7d9-4e82-9c0d-0f83f48c07a3' to '/home/clive/work/mlops/fork-tempo/docs/examples/explainer/artifacts/explainer/environment.tar.gz'\n", + "[########################################] | 100% Completed | 1min 13.1s\n" ] } ], @@ -359,7 +359,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "['Marital Status = Separated', 'Sex = Female', 'Capital Gain <= 0.00', 'Education = Associates', 'Age > 28.00']\n" + "['Marital Status = Separated', 'Sex = Female', 'Capital Gain <= 0.00', 'Education = Associates', 'Country = United-States']\n" ] } ], diff --git a/docs/examples/explainer/README.md b/docs/examples/explainer/README.md index 15b497b0..2439395c 100644 --- a/docs/examples/explainer/README.md +++ b/docs/examples/explainer/README.md @@ -195,8 +195,8 @@ tempo.save(Explainer) ``` Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-27d7d340-70df-4095-92b7-b9ef722eda26' to '/home/clive/work/mlops/fork-tempo/docs/examples/explainer/artifacts/explainer/environment.tar.gz' - [########################################] | 100% Completed | 59.8s + Packing environment at '/home/clive/anaconda3/envs/tempo-d87b2b65-e7d9-4e82-9c0d-0f83f48c07a3' to '/home/clive/work/mlops/fork-tempo/docs/examples/explainer/artifacts/explainer/environment.tar.gz' + [########################################] | 100% Completed | 1min 13.1s ## Test Locally on Docker @@ -224,7 +224,7 @@ r = json.loads(remote_model.predict(payload=data.X_test[0:1], parameters={"thres print(r["data"]["anchor"]) ``` - ['Marital Status = Separated', 'Sex = Female', 'Capital Gain <= 0.00', 'Education = Associates', 'Age > 28.00'] + ['Marital Status = Separated', 'Sex = Female', 'Capital Gain <= 0.00', 'Education = Associates', 'Country = United-States'] diff --git a/docs/examples/kfserving/README.ipynb b/docs/examples/kfserving/README.ipynb deleted file mode 100644 index 6185418a..00000000 --- a/docs/examples/kfserving/README.ipynb +++ /dev/null @@ -1,822 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8ef410d1", - "metadata": {}, - "source": [ - "# Deploy to KFserving\n", - "\n", - "![architecture](architecture.png)\n", - "\n", - "In this introduction we will:\n", - "\n", - " * [Describe the project structure](#Project-Structure)\n", - " * [Train some models](#Train-Models)\n", - " * [Create Tempo artifacts](#Create-Tempo-Artifacts)\n", - " * [Run unit tests](#Unit-Tests)\n", - " * [Save python environment for our classifier](#Save-Classifier-Environment)\n", - " * [Test Locally on Docker](#Test-Locally-on-Docker)\n", - " * [Production on Kubernetes via Tempo](#Production-Option-1-(Deploy-to-Kubernetes-with-Tempo))\n", - " * [Prodiuction on Kuebrnetes via GitOps](#Production-Option-2-(Gitops))" - ] - }, - { - "cell_type": "markdown", - "id": "c86b5277", - "metadata": {}, - "source": [ - "## Prerequisites\n", - "\n", - "This notebooks needs to be run in the `tempo-examples` conda environment defined below. Create from project root folder:\n", - "\n", - "```bash\n", - "conda env create --name tempo-examples --file conda/tempo-examples.yaml\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "f12ce5e8", - "metadata": {}, - "source": [ - "## Project Structure" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b8ca70f9", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[01;34m.\u001b[00m\r\n", - "├── \u001b[01;34martifacts\u001b[00m\r\n", - "│   ├── \u001b[01;34mclassifier\u001b[00m\r\n", - "│   ├── \u001b[01;34msklearn\u001b[00m\r\n", - "│   └── \u001b[01;34mxgboost\u001b[00m\r\n", - "├── \u001b[01;34mk8s\u001b[00m\r\n", - "│   └── \u001b[01;34mrbac\u001b[00m\r\n", - "├── \u001b[01;34msrc\u001b[00m\r\n", - "│   ├── constants.py\r\n", - "│   ├── data.py\r\n", - "│   ├── tempo.py\r\n", - "│   └── train.py\r\n", - "└── \u001b[01;34mtests\u001b[00m\r\n", - " └── test_deploy.py\r\n", - "\r\n", - "8 directories, 5 files\r\n" - ] - } - ], - "source": [ - "!tree -P \"*.py\" -I \"__init__.py|__pycache__\" -L 2" - ] - }, - { - "cell_type": "markdown", - "id": "b55dad4f", - "metadata": {}, - "source": [ - "## Train Models\n", - "\n", - " * This section is where as a data scientist you do your work of training models and creating artfacts.\n", - " * For this example we train sklearn and xgboost classification models for the iris dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "42c20ffa", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from tempo.utils import logger\n", - "import logging\n", - "import numpy as np\n", - "logger.setLevel(logging.ERROR)\n", - "logging.basicConfig(level=logging.ERROR)\n", - "ARTIFACTS_FOLDER = os.getcwd()+\"/artifacts\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "2e2abd28", - "metadata": { - "code_folding": [ - 0 - ] - }, - "outputs": [], - "source": [ - "# %load src/train.py\n", - "from typing import Tuple\n", - "\n", - "import joblib\n", - "import numpy as np\n", - "from sklearn import datasets\n", - "from sklearn.linear_model import LogisticRegression\n", - "from xgboost import XGBClassifier\n", - "\n", - "SKLearnFolder = \"sklearn\"\n", - "XGBoostFolder = \"xgboost\"\n", - "\n", - "\n", - "def load_iris() -> Tuple[np.ndarray, np.ndarray]:\n", - " iris = datasets.load_iris()\n", - " X = iris.data # we only take the first two features.\n", - " y = iris.target\n", - " return (X, y)\n", - "\n", - "\n", - "def train_sklearn(X: np.ndarray, y: np.ndarray, artifacts_folder: str):\n", - " logreg = LogisticRegression(C=1e5)\n", - " logreg.fit(X, y)\n", - " logreg.predict_proba(X[0:1])\n", - " with open(f\"{artifacts_folder}/{SKLearnFolder}/model.joblib\", \"wb\") as f:\n", - " joblib.dump(logreg, f)\n", - "\n", - "\n", - "def train_xgboost(X: np.ndarray, y: np.ndarray, artifacts_folder: str):\n", - " clf = XGBClassifier()\n", - " clf.fit(X, y)\n", - " clf.save_model(f\"{artifacts_folder}/{XGBoostFolder}/model.bst\")\n" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "aa35a350", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[11:51:11] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/clive/anaconda3/envs/tempo-dev/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n", - " warnings.warn(label_encoder_deprecation_msg, UserWarning)\n" - ] - } - ], - "source": [ - "from src.data import IrisData\n", - "from src.train import train_lr, train_xgb\n", - "data = IrisData()\n", - "\n", - "train_lr(ARTIFACTS_FOLDER, data)\n", - "train_xgb(ARTIFACTS_FOLDER, data)" - ] - }, - { - "cell_type": "markdown", - "id": "c396b609", - "metadata": {}, - "source": [ - "## Create Tempo Artifacts\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "a8345fb6", - "metadata": {}, - "outputs": [], - "source": [ - "from src.tempo import get_tempo_artifacts\n", - "classifier, sklearn_model, xgboost_model = get_tempo_artifacts(ARTIFACTS_FOLDER)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "c0b0af26", - "metadata": { - "code_folding": [ - 0 - ] - }, - "outputs": [], - "source": [ - "# %load src/tempo.py\n", - "from typing import Tuple\n", - "\n", - "import numpy as np\n", - "from src.constants import SKLearnFolder, XGBFolder, SKLearnTag, XGBoostTag\n", - "\n", - "from tempo.serve.metadata import ModelFramework\n", - "from tempo.serve.model import Model\n", - "from tempo.serve.pipeline import Pipeline, PipelineModels\n", - "from tempo.serve.utils import pipeline\n", - "\n", - "\n", - "def get_tempo_artifacts(artifacts_folder: str) -> Tuple[Pipeline, Model, Model]:\n", - " sklearn_model = Model(\n", - " name=\"test-iris-sklearn\",\n", - " platform=ModelFramework.SKLearn,\n", - " local_folder=f\"{artifacts_folder}/{SKLearnFolder}\",\n", - " uri=\"s3://tempo/basic/sklearn\",\n", - " description=\"SKLearn Iris classification model\",\n", - " )\n", - "\n", - " xgboost_model = Model(\n", - " name=\"test-iris-xgboost\",\n", - " platform=ModelFramework.XGBoost,\n", - " local_folder=f\"{artifacts_folder}/{XGBFolder}\",\n", - " uri=\"s3://tempo/basic/xgboost\",\n", - " description=\"XGBoost Iris classification model\",\n", - " )\n", - "\n", - " @pipeline(\n", - " name=\"classifier\",\n", - " uri=\"s3://tempo/basic/pipeline\",\n", - " local_folder=f\"{artifacts_folder}/classifier\",\n", - " models=PipelineModels(sklearn=sklearn_model, xgboost=xgboost_model),\n", - " description=\"A pipeline to use either an sklearn or xgboost model for Iris classification\",\n", - " )\n", - " def classifier(payload: np.ndarray) -> Tuple[np.ndarray, str]:\n", - " res1 = classifier.models.sklearn(input=payload)\n", - " print(res1)\n", - " if res1[0] == 1:\n", - " return res1, SKLearnTag\n", - " else:\n", - " return classifier.models.xgboost(input=payload), XGBoostTag\n", - "\n", - " return classifier, sklearn_model, xgboost_model\n" - ] - }, - { - "cell_type": "markdown", - "id": "f7a0525e", - "metadata": {}, - "source": [ - "## Unit Tests\n", - "\n", - " * Here we run our unit tests to ensure the orchestration works before running on the actual models." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "8159cbec", - "metadata": { - "code_folding": [ - 0 - ] - }, - "outputs": [], - "source": [ - "# %load tests/test_deploy.py\n", - "import numpy as np\n", - "from src.tempo import get_tempo_artifacts\n", - "from src.constants import SKLearnTag, XGBoostTag\n", - "\n", - "\n", - "def test_sklearn_model_used():\n", - " classifier, _, _ = get_tempo_artifacts(\"\")\n", - " classifier.models.sklearn = lambda input: np.array([[1]])\n", - " res, tag = classifier(np.array([[1, 2, 3, 4]]))\n", - " assert res[0][0] == 1\n", - " assert tag == SKLearnTag\n", - "\n", - "\n", - "def test_xgboost_model_used():\n", - " classifier, _, _ = get_tempo_artifacts(\"\")\n", - " classifier.models.sklearn = lambda input: np.array([[0.2]])\n", - " classifier.models.xgboost = lambda input: np.array([[0.1]])\n", - " res, tag = classifier(np.array([[1, 2, 3, 4]]))\n", - " assert res[0][0] == 0.1\n", - " assert tag == XGBoostTag\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "aa78ec19", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1m============================= test session starts ==============================\u001b[0m\n", - "platform linux -- Python 3.7.10, pytest-6.2.0, py-1.10.0, pluggy-0.13.1\n", - "rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg\n", - "plugins: cases-3.4.6, asyncio-0.14.0\n", - "collected 2 items \u001b[0m\u001b[1m\n", - "\n", - "tests/test_deploy.py \u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", - "\n", - "\u001b[32m============================== \u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 0.75s\u001b[0m\u001b[32m ===============================\u001b[0m\n" - ] - } - ], - "source": [ - "!python -m pytest tests/" - ] - }, - { - "cell_type": "markdown", - "id": "18dc6a4b", - "metadata": {}, - "source": [ - "## Save Classifier Environment\n", - "\n", - " * In preparation for running our models we save the Python environment needed for the orchestration to run as defined by a `conda.yaml` in our project." - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "id": "3e1f9017", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "artifacts/classifier/conda.yaml\r\n" - ] - } - ], - "source": [ - "!ls artifacts/classifier/conda.yaml" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "3c23ab3b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-b078d4e0-48a7-4c6e-bf46-74fc623ea46a' to '/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/classifier/environment.tar.gz'\n", - "[########################################] | 100% Completed | 13.7s\n" - ] - } - ], - "source": [ - "from tempo.serve.loader import save\n", - "save(classifier)" - ] - }, - { - "cell_type": "markdown", - "id": "ae7a21d5", - "metadata": {}, - "source": [ - "## Test Locally on Docker\n", - "\n", - " * Here we test our models using production images but running locally on Docker. This allows us to ensure the final production deployed model will behave as expected when deployed." - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "id": "a36bfb9f", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo import deploy_local\n", - "remote_model = deploy_local(classifier)" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "6fad9e5e", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'}\n", - "{'output0': array([[0.97329617, 0.02412145, 0.00258233]], dtype=float32), 'output1': 'xgboost prediction'}\n" - ] - } - ], - "source": [ - "print(remote_model.predict(np.array([[0, 0, 0,0]])))\n", - "print(remote_model.predict(np.array([[5.964,4.006,2.081,1.031]])))" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "6c6ea7c7", - "metadata": {}, - "outputs": [], - "source": [ - "remote_model.undeploy()" - ] - }, - { - "cell_type": "markdown", - "id": "7a91f4d7", - "metadata": {}, - "source": [ - "## Production Option 1 (Deploy to Kubernetes with Tempo)\n", - "\n", - " * Here we illustrate how to run the final models in \"production\" on Kubernetes by using Tempo to deploy\n", - " \n", - "### Prerequisites\n", - " \n", - "Create a Kind Kubernetes cluster with Minio and KFserving installed using Ansible as described [here](https://tempo.readthedocs.io/en/latest/overview/quickstart.html#kubernetes-cluster-with-kfserving)." - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "eacfd8cc", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Error from server (AlreadyExists): namespaces \"production\" already exists\r\n" - ] - } - ], - "source": [ - "!kubectl create ns production" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "d8d2fb32", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "secret/minio-secret configured\r\n", - "serviceaccount/kf-tempo configured\r\n", - "role.rbac.authorization.k8s.io/kf-tempo unchanged\r\n", - "rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding unchanged\r\n" - ] - } - ], - "source": [ - "!kubectl apply -f k8s/rbac -n production" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "9fa80565", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo.examples.minio import create_minio_rclone\n", - "import os\n", - "create_minio_rclone(os.getcwd()+\"/rclone.conf\")" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "39ff404c", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo.serve.loader import upload\n", - "upload(sklearn_model)\n", - "upload(xgboost_model)\n", - "upload(classifier)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "5b3b03e6", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo.serve.metadata import SeldonCoreOptions\n", - "runtime_options = SeldonCoreOptions(**{\n", - " \"remote_options\": {\n", - " \"runtime\": \"tempo.kfserving.KFServingKubernetesRuntime\",\n", - " \"namespace\": \"production\",\n", - " \"serviceAccountName\": \"kf-tempo\"\n", - " }\n", - " })" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "363a9b61", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo import deploy_remote\n", - "remote_model = deploy_remote(classifier, options=runtime_options)" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "8feb662b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "{'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'}\n", - "{'output0': array([[0.00847207, 0.03168794, 0.95984 ]], dtype=float32), 'output1': 'xgboost prediction'}\n" - ] - } - ], - "source": [ - "print(remote_model.predict(payload=np.array([[0, 0, 0, 0]])))\n", - "print(remote_model.predict(payload=np.array([[1, 2, 3, 4]])))" - ] - }, - { - "cell_type": "markdown", - "id": "3b8e8ca4", - "metadata": {}, - "source": [ - "### Illustrate client using model remotely\n", - "\n", - "With the Kubernetes runtime one can list running models on the Kubernetes cluster and instantiate a RemoteModel to call the Tempo model." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "adc7dc37", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Name\tDescription\n", - "classifier\tA pipeline to use either an sklearn or xgboost model for Iris classification\n", - "test-iris-sklearn\tSKLearn Iris classification model\n", - "test-iris-xgboost\tXGBoost Iris classification model\n" - ] - } - ], - "source": [ - "from tempo.kfserving.k8s import KFServingKubernetesRuntime\n", - "k8s_runtime = KFServingKubernetesRuntime(runtime_options.remote_options)\n", - "models = k8s_runtime.list_models(namespace=\"production\")\n", - "print(\"Name\\tDescription\")\n", - "for model in models:\n", - " details = model.get_tempo().model_spec.model_details\n", - " print(f\"{details.name}\\t{details.description}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "3e1903eb", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'output0': array([[0.00847207, 0.03168794, 0.95984 ]], dtype=float32),\n", - " 'output1': 'xgboost prediction'}" - ] - }, - "execution_count": 27, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "models[0].predict(payload=np.array([[1, 2, 3, 4]]))" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "id": "0c5b7b61", - "metadata": {}, - "outputs": [], - "source": [ - "remote_model.undeploy()" - ] - }, - { - "cell_type": "markdown", - "id": "efd7205b", - "metadata": {}, - "source": [ - "## Production Option 2 (Gitops)\n", - "\n", - " * We create yaml to provide to our DevOps team to deploy to a production cluster\n", - " * We add Kustomize patches to modify the base Kubernetes yaml created by Tempo" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "id": "69ee5f4a", - "metadata": {}, - "outputs": [], - "source": [ - "from tempo import manifest\n", - "from tempo.serve.metadata import SeldonCoreOptions\n", - "runtime_options = SeldonCoreOptions(**{\n", - " \"remote_options\": {\n", - " \"runtime\": \"tempo.kfserving.KFServingKubernetesRuntime\",\n", - " \"namespace\": \"production\",\n", - " \"serviceAccountName\": \"kf-tempo\"\n", - " }\n", - " })\n", - "yaml_str = manifest(classifier, options=runtime_options)\n", - "with open(os.getcwd()+\"/k8s/tempo.yaml\",\"w\") as f:\n", - " f.write(yaml_str)" - ] - }, - { - "cell_type": "code", - "execution_count": 30, - "id": "748bd754", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "apiVersion: serving.kubeflow.org/v1beta1\r\n", - "kind: InferenceService\r\n", - "metadata:\r\n", - " annotations:\r\n", - " seldon.io/tempo-description: A pipeline to use either an sklearn or xgboost model\r\n", - " for Iris classification\r\n", - " seldon.io/tempo-model: '{\"model_details\": {\"name\": \"classifier\", \"local_folder\":\r\n", - " \"/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/classifier\",\r\n", - " \"uri\": \"s3://tempo/basic/pipeline\", \"platform\": \"tempo\", \"inputs\": {\"args\":\r\n", - " [{\"ty\": \"numpy.ndarray\", \"name\": \"payload\"}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", - " \"name\": null}, {\"ty\": \"builtins.str\", \"name\": null}]}, \"description\": \"A pipeline\r\n", - " to use either an sklearn or xgboost model for Iris classification\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.kfserving.KFServingKubernetesRuntime\", \"state_options\": {\"state_type\":\r\n", - " \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": null, \"serviceAccountName\": \"kf-tempo\", \"add_svc_orchestrator\":\r\n", - " false, \"namespace\": \"production\"}}'\r\n", - " labels:\r\n", - " seldon.io/tempo: \"true\"\r\n", - " name: classifier\r\n", - " namespace: production\r\n", - "spec:\r\n", - " predictor:\r\n", - " containers:\r\n", - " - env:\r\n", - " - name: STORAGE_URI\r\n", - " value: s3://tempo/basic/pipeline\r\n", - " - name: MLSERVER_HTTP_PORT\r\n", - " value: \"8080\"\r\n", - " - name: MLSERVER_GRPC_PORT\r\n", - " value: \"9000\"\r\n", - " - name: MLSERVER_MODEL_IMPLEMENTATION\r\n", - " value: tempo.mlserver.InferenceRuntime\r\n", - " - name: MLSERVER_MODEL_NAME\r\n", - " value: classifier\r\n", - " - name: MLSERVER_MODEL_URI\r\n", - " value: /mnt/models\r\n", - " - name: TEMPO_RUNTIME_OPTIONS\r\n", - " value: '{\"runtime\": \"tempo.kfserving.KFServingKubernetesRuntime\", \"state_options\":\r\n", - " {\"state_type\": \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\":\r\n", - " {\"worker_endpoint\": \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3,\r\n", - " \"window_time\": 0, \"mode_type\": \"NONE\", \"in_asyncio\": false}, \"ingress_options\":\r\n", - " {\"ingress\": \"tempo.ingress.istio.IstioIngress\", \"ssl\": false, \"verify_ssl\":\r\n", - " true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\": null, \"authSecretName\":\r\n", - " null, \"serviceAccountName\": \"kf-tempo\", \"add_svc_orchestrator\": false, \"namespace\":\r\n", - " \"production\"}'\r\n", - " image: seldonio/mlserver:0.3.2\r\n", - " name: mlserver\r\n", - " resources:\r\n", - " limits:\r\n", - " cpu: 1\r\n", - " memory: 1Gi\r\n", - " requests:\r\n", - " cpu: 500m\r\n", - " memory: 500Mi\r\n", - " serviceAccountName: kf-tempo\r\n", - "---\r\n", - "apiVersion: serving.kubeflow.org/v1beta1\r\n", - "kind: InferenceService\r\n", - "metadata:\r\n", - " annotations:\r\n", - " seldon.io/tempo-description: SKLearn Iris classification model\r\n", - " seldon.io/tempo-model: '{\"model_details\": {\"name\": \"test-iris-sklearn\", \"local_folder\":\r\n", - " \"/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/sklearn\",\r\n", - " \"uri\": \"s3://tempo/basic/sklearn\", \"platform\": \"sklearn\", \"inputs\": {\"args\":\r\n", - " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", - " \"name\": null}]}, \"description\": \"SKLearn Iris classification model\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.kfserving.KFServingKubernetesRuntime\", \"state_options\": {\"state_type\":\r\n", - " \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": null, \"serviceAccountName\": \"kf-tempo\", \"add_svc_orchestrator\":\r\n", - " false, \"namespace\": \"production\"}}'\r\n", - " labels:\r\n", - " seldon.io/tempo: \"true\"\r\n", - " name: test-iris-sklearn\r\n", - " namespace: production\r\n", - "spec:\r\n", - " predictor:\r\n", - " serviceAccountName: kf-tempo\r\n", - " sklearn:\r\n", - " protocolVersion: v2\r\n", - " storageUri: s3://tempo/basic/sklearn\r\n", - "---\r\n", - "apiVersion: serving.kubeflow.org/v1beta1\r\n", - "kind: InferenceService\r\n", - "metadata:\r\n", - " annotations:\r\n", - " seldon.io/tempo-description: XGBoost Iris classification model\r\n", - " seldon.io/tempo-model: '{\"model_details\": {\"name\": \"test-iris-xgboost\", \"local_folder\":\r\n", - " \"/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/xgboost\",\r\n", - " \"uri\": \"s3://tempo/basic/xgboost\", \"platform\": \"xgboost\", \"inputs\": {\"args\":\r\n", - " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", - " \"name\": null}]}, \"description\": \"XGBoost Iris classification model\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.kfserving.KFServingKubernetesRuntime\", \"state_options\": {\"state_type\":\r\n", - " \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": null, \"serviceAccountName\": \"kf-tempo\", \"add_svc_orchestrator\":\r\n", - " false, \"namespace\": \"production\"}}'\r\n", - " labels:\r\n", - " seldon.io/tempo: \"true\"\r\n", - " name: test-iris-xgboost\r\n", - " namespace: production\r\n", - "spec:\r\n", - " predictor:\r\n", - " serviceAccountName: kf-tempo\r\n", - " xgboost:\r\n", - " protocolVersion: v2\r\n", - " storageUri: s3://tempo/basic/xgboost\r\n" - ] - } - ], - "source": [ - "!kustomize build k8s" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11fff2db", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.10" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/docs/examples/kfserving/README.md b/docs/examples/kfserving/README.md deleted file mode 100644 index 91cb57f1..00000000 --- a/docs/examples/kfserving/README.md +++ /dev/null @@ -1,529 +0,0 @@ -# Deploy to KFserving - -![architecture](architecture.png) - -In this introduction we will: - - * [Describe the project structure](#Project-Structure) - * [Train some models](#Train-Models) - * [Create Tempo artifacts](#Create-Tempo-Artifacts) - * [Run unit tests](#Unit-Tests) - * [Save python environment for our classifier](#Save-Classifier-Environment) - * [Test Locally on Docker](#Test-Locally-on-Docker) - * [Production on Kubernetes via Tempo](#Production-Option-1-(Deploy-to-Kubernetes-with-Tempo)) - * [Prodiuction on Kuebrnetes via GitOps](#Production-Option-2-(Gitops)) - -## Prerequisites - -This notebooks needs to be run in the `tempo-examples` conda environment defined below. Create from project root folder: - -```bash -conda env create --name tempo-examples --file conda/tempo-examples.yaml -``` - -## Project Structure - - -```python -!tree -P "*.py" -I "__init__.py|__pycache__" -L 2 -``` - - . - ├── artifacts - │   ├── classifier - │   ├── sklearn - │   └── xgboost - ├── k8s - │   └── rbac - ├── src - │   ├── constants.py - │   ├── data.py - │   ├── tempo.py - │   └── train.py - └── tests - └── test_deploy.py - - 8 directories, 5 files - - -## Train Models - - * This section is where as a data scientist you do your work of training models and creating artfacts. - * For this example we train sklearn and xgboost classification models for the iris dataset. - - -```python -import os -from tempo.utils import logger -import logging -import numpy as np -logger.setLevel(logging.ERROR) -logging.basicConfig(level=logging.ERROR) -ARTIFACTS_FOLDER = os.getcwd()+"/artifacts" -``` - - -```python -# %load src/train.py -from typing import Tuple - -import joblib -import numpy as np -from sklearn import datasets -from sklearn.linear_model import LogisticRegression -from xgboost import XGBClassifier - -SKLearnFolder = "sklearn" -XGBoostFolder = "xgboost" - - -def load_iris() -> Tuple[np.ndarray, np.ndarray]: - iris = datasets.load_iris() - X = iris.data # we only take the first two features. - y = iris.target - return (X, y) - - -def train_sklearn(X: np.ndarray, y: np.ndarray, artifacts_folder: str): - logreg = LogisticRegression(C=1e5) - logreg.fit(X, y) - logreg.predict_proba(X[0:1]) - with open(f"{artifacts_folder}/{SKLearnFolder}/model.joblib", "wb") as f: - joblib.dump(logreg, f) - - -def train_xgboost(X: np.ndarray, y: np.ndarray, artifacts_folder: str): - clf = XGBClassifier() - clf.fit(X, y) - clf.save_model(f"{artifacts_folder}/{XGBoostFolder}/model.bst") - -``` - - -```python -from src.data import IrisData -from src.train import train_lr, train_xgb -data = IrisData() - -train_lr(ARTIFACTS_FOLDER, data) -train_xgb(ARTIFACTS_FOLDER, data) -``` - - [11:51:11] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior. - - - /home/clive/anaconda3/envs/tempo-dev/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1]. - warnings.warn(label_encoder_deprecation_msg, UserWarning) - - -## Create Tempo Artifacts - - - -```python -from src.tempo import get_tempo_artifacts -classifier, sklearn_model, xgboost_model = get_tempo_artifacts(ARTIFACTS_FOLDER) -``` - - -```python -# %load src/tempo.py -from typing import Tuple - -import numpy as np -from src.constants import SKLearnFolder, XGBFolder, SKLearnTag, XGBoostTag - -from tempo.serve.metadata import ModelFramework -from tempo.serve.model import Model -from tempo.serve.pipeline import Pipeline, PipelineModels -from tempo.serve.utils import pipeline - - -def get_tempo_artifacts(artifacts_folder: str) -> Tuple[Pipeline, Model, Model]: - sklearn_model = Model( - name="test-iris-sklearn", - platform=ModelFramework.SKLearn, - local_folder=f"{artifacts_folder}/{SKLearnFolder}", - uri="s3://tempo/basic/sklearn", - description="SKLearn Iris classification model", - ) - - xgboost_model = Model( - name="test-iris-xgboost", - platform=ModelFramework.XGBoost, - local_folder=f"{artifacts_folder}/{XGBFolder}", - uri="s3://tempo/basic/xgboost", - description="XGBoost Iris classification model", - ) - - @pipeline( - name="classifier", - uri="s3://tempo/basic/pipeline", - local_folder=f"{artifacts_folder}/classifier", - models=PipelineModels(sklearn=sklearn_model, xgboost=xgboost_model), - description="A pipeline to use either an sklearn or xgboost model for Iris classification", - ) - def classifier(payload: np.ndarray) -> Tuple[np.ndarray, str]: - res1 = classifier.models.sklearn(input=payload) - print(res1) - if res1[0] == 1: - return res1, SKLearnTag - else: - return classifier.models.xgboost(input=payload), XGBoostTag - - return classifier, sklearn_model, xgboost_model - -``` - -## Unit Tests - - * Here we run our unit tests to ensure the orchestration works before running on the actual models. - - -```python -# %load tests/test_deploy.py -import numpy as np -from src.tempo import get_tempo_artifacts -from src.constants import SKLearnTag, XGBoostTag - - -def test_sklearn_model_used(): - classifier, _, _ = get_tempo_artifacts("") - classifier.models.sklearn = lambda input: np.array([[1]]) - res, tag = classifier(np.array([[1, 2, 3, 4]])) - assert res[0][0] == 1 - assert tag == SKLearnTag - - -def test_xgboost_model_used(): - classifier, _, _ = get_tempo_artifacts("") - classifier.models.sklearn = lambda input: np.array([[0.2]]) - classifier.models.xgboost = lambda input: np.array([[0.1]]) - res, tag = classifier(np.array([[1, 2, 3, 4]])) - assert res[0][0] == 0.1 - assert tag == XGBoostTag - -``` - - -```python -!python -m pytest tests/ -``` - - ============================= test session starts ============================== - platform linux -- Python 3.7.10, pytest-6.2.0, py-1.10.0, pluggy-0.13.1 - rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg - plugins: cases-3.4.6, asyncio-0.14.0 - collected 2 items  - - tests/test_deploy.py .. [100%] - - ============================== 2 passed in 0.75s =============================== - - -## Save Classifier Environment - - * In preparation for running our models we save the Python environment needed for the orchestration to run as defined by a `conda.yaml` in our project. - - -```python -!ls artifacts/classifier/conda.yaml -``` - - artifacts/classifier/conda.yaml - - - -```python -from tempo.serve.loader import save -save(classifier) -``` - - Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-b078d4e0-48a7-4c6e-bf46-74fc623ea46a' to '/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/classifier/environment.tar.gz' - [########################################] | 100% Completed | 13.7s - - -## Test Locally on Docker - - * Here we test our models using production images but running locally on Docker. This allows us to ensure the final production deployed model will behave as expected when deployed. - - -```python -from tempo import deploy_local -remote_model = deploy_local(classifier) -``` - - -```python -print(remote_model.predict(np.array([[0, 0, 0,0]]))) -print(remote_model.predict(np.array([[5.964,4.006,2.081,1.031]]))) -``` - - {'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'} - {'output0': array([[0.97329617, 0.02412145, 0.00258233]], dtype=float32), 'output1': 'xgboost prediction'} - - - -```python -remote_model.undeploy() -``` - -## Production Option 1 (Deploy to Kubernetes with Tempo) - - * Here we illustrate how to run the final models in "production" on Kubernetes by using Tempo to deploy - -### Prerequisites - -Create a Kind Kubernetes cluster with Minio and KFserving installed using Ansible as described [here](https://tempo.readthedocs.io/en/latest/overview/quickstart.html#kubernetes-cluster-with-kfserving). - - -```python -!kubectl create ns production -``` - - Error from server (AlreadyExists): namespaces "production" already exists - - - -```python -!kubectl apply -f k8s/rbac -n production -``` - - secret/minio-secret configured - serviceaccount/kf-tempo configured - role.rbac.authorization.k8s.io/kf-tempo unchanged - rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding unchanged - - - -```python -from tempo.examples.minio import create_minio_rclone -import os -create_minio_rclone(os.getcwd()+"/rclone.conf") -``` - - -```python -from tempo.serve.loader import upload -upload(sklearn_model) -upload(xgboost_model) -upload(classifier) -``` - - -```python -from tempo.serve.metadata import SeldonCoreOptions -runtime_options = SeldonCoreOptions(**{ - "remote_options": { - "runtime": "tempo.kfserving.KFServingKubernetesRuntime", - "namespace": "production", - "serviceAccountName": "kf-tempo" - } - }) -``` - - -```python -from tempo import deploy_remote -remote_model = deploy_remote(classifier, options=runtime_options) -``` - - -```python -print(remote_model.predict(payload=np.array([[0, 0, 0, 0]]))) -print(remote_model.predict(payload=np.array([[1, 2, 3, 4]]))) -``` - - {'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'} - {'output0': array([[0.00847207, 0.03168794, 0.95984 ]], dtype=float32), 'output1': 'xgboost prediction'} - - -### Illustrate client using model remotely - -With the Kubernetes runtime one can list running models on the Kubernetes cluster and instantiate a RemoteModel to call the Tempo model. - - -```python -from tempo.kfserving.k8s import KFServingKubernetesRuntime -k8s_runtime = KFServingKubernetesRuntime(runtime_options.remote_options) -models = k8s_runtime.list_models(namespace="production") -print("Name\tDescription") -for model in models: - details = model.get_tempo().model_spec.model_details - print(f"{details.name}\t{details.description}") -``` - - Name Description - classifier A pipeline to use either an sklearn or xgboost model for Iris classification - test-iris-sklearn SKLearn Iris classification model - test-iris-xgboost XGBoost Iris classification model - - - -```python -models[0].predict(payload=np.array([[1, 2, 3, 4]])) -``` - - - - - {'output0': array([[0.00847207, 0.03168794, 0.95984 ]], dtype=float32), - 'output1': 'xgboost prediction'} - - - - -```python -remote_model.undeploy() -``` - -## Production Option 2 (Gitops) - - * We create yaml to provide to our DevOps team to deploy to a production cluster - * We add Kustomize patches to modify the base Kubernetes yaml created by Tempo - - -```python -from tempo import manifest -from tempo.serve.metadata import SeldonCoreOptions -runtime_options = SeldonCoreOptions(**{ - "remote_options": { - "runtime": "tempo.kfserving.KFServingKubernetesRuntime", - "namespace": "production", - "serviceAccountName": "kf-tempo" - } - }) -yaml_str = manifest(classifier, options=runtime_options) -with open(os.getcwd()+"/k8s/tempo.yaml","w") as f: - f.write(yaml_str) -``` - - -```python -!kustomize build k8s -``` - - apiVersion: serving.kubeflow.org/v1beta1 - kind: InferenceService - metadata: - annotations: - seldon.io/tempo-description: A pipeline to use either an sklearn or xgboost model - for Iris classification - seldon.io/tempo-model: '{"model_details": {"name": "classifier", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/classifier", - "uri": "s3://tempo/basic/pipeline", "platform": "tempo", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": "payload"}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}, {"ty": "builtins.str", "name": null}]}, "description": "A pipeline - to use either an sklearn or xgboost model for Iris classification"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: "true" - name: classifier - namespace: production - spec: - predictor: - containers: - - env: - - name: STORAGE_URI - value: s3://tempo/basic/pipeline - - name: MLSERVER_HTTP_PORT - value: "8080" - - name: MLSERVER_GRPC_PORT - value: "9000" - - name: MLSERVER_MODEL_IMPLEMENTATION - value: tempo.mlserver.InferenceRuntime - - name: MLSERVER_MODEL_NAME - value: classifier - - name: MLSERVER_MODEL_URI - value: /mnt/models - - name: TEMPO_RUNTIME_OPTIONS - value: '{"runtime": "tempo.kfserving.KFServingKubernetesRuntime", "state_options": - {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": - {"worker_endpoint": "", "batch_size": 1, "parallelism": 1, "retries": 3, - "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, "ingress_options": - {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, "verify_ssl": - true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, "authSecretName": - null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": false, "namespace": - "production"}' - image: seldonio/mlserver:0.3.2 - name: mlserver - resources: - limits: - cpu: 1 - memory: 1Gi - requests: - cpu: 500m - memory: 500Mi - serviceAccountName: kf-tempo - --- - apiVersion: serving.kubeflow.org/v1beta1 - kind: InferenceService - metadata: - annotations: - seldon.io/tempo-description: SKLearn Iris classification model - seldon.io/tempo-model: '{"model_details": {"name": "test-iris-sklearn", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/sklearn", - "uri": "s3://tempo/basic/sklearn", "platform": "sklearn", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}]}, "description": "SKLearn Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: "true" - name: test-iris-sklearn - namespace: production - spec: - predictor: - serviceAccountName: kf-tempo - sklearn: - protocolVersion: v2 - storageUri: s3://tempo/basic/sklearn - --- - apiVersion: serving.kubeflow.org/v1beta1 - kind: InferenceService - metadata: - annotations: - seldon.io/tempo-description: XGBoost Iris classification model - seldon.io/tempo-model: '{"model_details": {"name": "test-iris-xgboost", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/xgboost", - "uri": "s3://tempo/basic/xgboost", "platform": "xgboost", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}]}, "description": "XGBoost Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: "true" - name: test-iris-xgboost - namespace: production - spec: - predictor: - serviceAccountName: kf-tempo - xgboost: - protocolVersion: v2 - storageUri: s3://tempo/basic/xgboost - - - -```python - -``` diff --git a/docs/examples/kfserving/architecture.png b/docs/examples/kfserving/architecture.png deleted file mode 100644 index f5758940..00000000 Binary files a/docs/examples/kfserving/architecture.png and /dev/null differ diff --git a/docs/examples/kfserving/artifacts/classifier/.gitignore b/docs/examples/kfserving/artifacts/classifier/.gitignore deleted file mode 100644 index 83757ea5..00000000 --- a/docs/examples/kfserving/artifacts/classifier/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ - environment.tar.gz - model.pickle \ No newline at end of file diff --git a/docs/examples/kfserving/artifacts/classifier/conda.yaml b/docs/examples/kfserving/artifacts/classifier/conda.yaml deleted file mode 100644 index 7dd962e5..00000000 --- a/docs/examples/kfserving/artifacts/classifier/conda.yaml +++ /dev/null @@ -1,8 +0,0 @@ -name: tempo -channels: - - defaults -dependencies: - - python=3.7.9 - - pip: - - mlops-tempo - - mlserver==0.4.0 diff --git a/docs/examples/kfserving/artifacts/sklearn/.gitignore b/docs/examples/kfserving/artifacts/sklearn/.gitignore deleted file mode 100644 index d5ff1a1d..00000000 --- a/docs/examples/kfserving/artifacts/sklearn/.gitignore +++ /dev/null @@ -1 +0,0 @@ -model.joblib diff --git a/docs/examples/kfserving/artifacts/xgboost/.gitignore b/docs/examples/kfserving/artifacts/xgboost/.gitignore deleted file mode 100644 index 5e165ee2..00000000 --- a/docs/examples/kfserving/artifacts/xgboost/.gitignore +++ /dev/null @@ -1 +0,0 @@ -model.bst diff --git a/docs/examples/kfserving/k8s/kustomization.yaml b/docs/examples/kfserving/k8s/kustomization.yaml deleted file mode 100644 index 2ecba571..00000000 --- a/docs/examples/kfserving/k8s/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Adds namespace to all resources. -namespace: production - -resources: -- tempo.yaml - - -patchesJson6902: -- target: - group: serving.kubeflow.org - version: v1beta1 - kind: InferenceService - name: classifier - path: patch_resources.yaml - - - diff --git a/docs/examples/kfserving/k8s/patch_resources.yaml b/docs/examples/kfserving/k8s/patch_resources.yaml deleted file mode 100644 index 4b6f4346..00000000 --- a/docs/examples/kfserving/k8s/patch_resources.yaml +++ /dev/null @@ -1,10 +0,0 @@ -- op: add - path: /spec/predictor/containers/0/resources - value: - requests: - cpu: 500m - memory: 500Mi - limits: - cpu: 1 - memory: 1Gi - diff --git a/docs/examples/kfserving/k8s/rbac/auth.yaml b/docs/examples/kfserving/k8s/rbac/auth.yaml deleted file mode 100644 index 7e177729..00000000 --- a/docs/examples/kfserving/k8s/rbac/auth.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio-secret - annotations: - serving.kubeflow.org/s3-endpoint: minio.minio-system.svc.cluster.local:9000 # replace with your s3 endpoint - serving.kubeflow.org/s3-usehttps: "0" # by default 1, for testing with minio you need to set to 0 -type: Opaque -stringData: - AWS_ACCESS_KEY_ID: minioadmin - AWS_SECRET_ACCESS_KEY: minioadmin ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kf-tempo -secrets: -- name: minio-secret ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: kf-tempo -rules: - - apiGroups: - - machinelearning.seldon.io - resources: - - seldondeployments/status - verbs: - - get - - apiGroups: - - serving.kubeflow.org - resources: - - inferenceservices/status - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: tempo-pipeline-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kf-tempo -subjects: - - kind: ServiceAccount - name: kf-tempo diff --git a/docs/examples/kfserving/k8s/tempo.yaml b/docs/examples/kfserving/k8s/tempo.yaml deleted file mode 100644 index 1320ff78..00000000 --- a/docs/examples/kfserving/k8s/tempo.yaml +++ /dev/null @@ -1,114 +0,0 @@ -apiVersion: serving.kubeflow.org/v1beta1 -kind: InferenceService -metadata: - annotations: - seldon.io/tempo-description: A pipeline to use either an sklearn or xgboost model - for Iris classification - seldon.io/tempo-model: '{"model_details": {"name": "classifier", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/classifier", - "uri": "s3://tempo/basic/pipeline", "platform": "tempo", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": "payload"}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}, {"ty": "builtins.str", "name": null}]}, "description": "A pipeline - to use either an sklearn or xgboost model for Iris classification"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: 'true' - name: classifier - namespace: production -spec: - predictor: - containers: - - env: - - name: STORAGE_URI - value: s3://tempo/basic/pipeline - - name: MLSERVER_HTTP_PORT - value: '8080' - - name: MLSERVER_GRPC_PORT - value: '9000' - - name: MLSERVER_MODEL_IMPLEMENTATION - value: tempo.mlserver.InferenceRuntime - - name: MLSERVER_MODEL_NAME - value: classifier - - name: MLSERVER_MODEL_URI - value: /mnt/models - - name: TEMPO_RUNTIME_OPTIONS - value: '{"runtime": "tempo.kfserving.KFServingKubernetesRuntime", "state_options": - {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": - {"worker_endpoint": "", "batch_size": 1, "parallelism": 1, "retries": 3, - "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, "ingress_options": - {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, "verify_ssl": - true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, "authSecretName": - null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": false, "namespace": - "production"}' - image: seldonio/mlserver:0.3.2 - name: mlserver - serviceAccountName: kf-tempo - ---- -apiVersion: serving.kubeflow.org/v1beta1 -kind: InferenceService -metadata: - annotations: - seldon.io/tempo-description: SKLearn Iris classification model - seldon.io/tempo-model: '{"model_details": {"name": "test-iris-sklearn", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/sklearn", - "uri": "s3://tempo/basic/sklearn", "platform": "sklearn", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}]}, "description": "SKLearn Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: 'true' - name: test-iris-sklearn - namespace: production -spec: - predictor: - serviceAccountName: kf-tempo - sklearn: - protocolVersion: v2 - storageUri: s3://tempo/basic/sklearn - ---- -apiVersion: serving.kubeflow.org/v1beta1 -kind: InferenceService -metadata: - annotations: - seldon.io/tempo-description: XGBoost Iris classification model - seldon.io/tempo-model: '{"model_details": {"name": "test-iris-xgboost", "local_folder": - "/home/clive/work/mlops/fork-tempo/docs/examples/kfserving/artifacts/xgboost", - "uri": "s3://tempo/basic/xgboost", "platform": "xgboost", "inputs": {"args": - [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", - "name": null}]}, "description": "XGBoost Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.kfserving.KFServingKubernetesRuntime", "state_options": {"state_type": - "LOCAL", "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": null, "serviceAccountName": "kf-tempo", "add_svc_orchestrator": - false, "namespace": "production"}}' - labels: - seldon.io/tempo: 'true' - name: test-iris-xgboost - namespace: production -spec: - predictor: - serviceAccountName: kf-tempo - xgboost: - protocolVersion: v2 - storageUri: s3://tempo/basic/xgboost - ---- diff --git a/docs/examples/kfserving/rclone.conf b/docs/examples/kfserving/rclone.conf deleted file mode 100644 index 4d7dda62..00000000 --- a/docs/examples/kfserving/rclone.conf +++ /dev/null @@ -1,8 +0,0 @@ - -[s3] -type = s3 -provider = minio -env_auth = false -access_key_id = minioadmin -secret_access_key = minioadmin -endpoint = http://172.18.255.2:9000 diff --git a/docs/examples/kfserving/setup.cfg b/docs/examples/kfserving/setup.cfg deleted file mode 100644 index 580ff98d..00000000 --- a/docs/examples/kfserving/setup.cfg +++ /dev/null @@ -1,32 +0,0 @@ -[flake8] -max-line-length = 120 -extend-ignore = - # See https://github.com/PyCQA/pycodestyle/issues/373 - E203, -exclude = - ./build - ./dist - ./*.egg-info - ./.eggs - ./.tox - -[mypy] -ignore_missing_imports = True -plugins = pydantic.mypy - -[isort] -# See https://black.readthedocs.io/en/stable/compatible_configs.html -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 120 - -[tox:tox] -basepython = py3 -envlist = py3 - -[testenv] -deps = -r{toxinidir}/requirements-dev.txt -commands = pytest {posargs} diff --git a/docs/examples/kfserving/src/constants.py b/docs/examples/kfserving/src/constants.py deleted file mode 100644 index 48556a09..00000000 --- a/docs/examples/kfserving/src/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -SKLearnFolder = "sklearn" -XGBFolder = "xgboost" -SKLearnTag = "sklearn prediction" -XGBoostTag = "xgboost prediction" diff --git a/docs/examples/kfserving/src/data.py b/docs/examples/kfserving/src/data.py deleted file mode 100644 index 84a46e55..00000000 --- a/docs/examples/kfserving/src/data.py +++ /dev/null @@ -1,8 +0,0 @@ -from sklearn import datasets - - -class IrisData(object): - def __init__(self): - iris = datasets.load_iris() - self.X = iris.data # we only take the first two features. - self.y = iris.target diff --git a/docs/examples/kfserving/src/tempo.py b/docs/examples/kfserving/src/tempo.py deleted file mode 100644 index 7e033f58..00000000 --- a/docs/examples/kfserving/src/tempo.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import Tuple - -import numpy as np -from src.constants import SKLearnFolder, SKLearnTag, XGBFolder, XGBoostTag - -from tempo.serve.metadata import ModelFramework -from tempo.serve.model import Model -from tempo.serve.pipeline import Pipeline, PipelineModels -from tempo.serve.utils import pipeline - - -def get_tempo_artifacts(artifacts_folder: str) -> Tuple[Pipeline, Model, Model]: - sklearn_model = Model( - name="test-iris-sklearn", - platform=ModelFramework.SKLearn, - local_folder=f"{artifacts_folder}/{SKLearnFolder}", - uri="s3://tempo/basic/sklearn", - description="SKLearn Iris classification model", - ) - - xgboost_model = Model( - name="test-iris-xgboost", - platform=ModelFramework.XGBoost, - local_folder=f"{artifacts_folder}/{XGBFolder}", - uri="s3://tempo/basic/xgboost", - description="XGBoost Iris classification model", - ) - - @pipeline( - name="classifier", - uri="s3://tempo/basic/pipeline", - local_folder=f"{artifacts_folder}/classifier", - models=PipelineModels(sklearn=sklearn_model, xgboost=xgboost_model), - description="A pipeline to use either an sklearn or xgboost model for Iris classification", - ) - def classifier(payload: np.ndarray) -> Tuple[np.ndarray, str]: - res1 = classifier.models.sklearn(input=payload) - print(res1) - if res1[0] == 1: - return res1, SKLearnTag - else: - return classifier.models.xgboost(input=payload), XGBoostTag - - return classifier, sklearn_model, xgboost_model diff --git a/docs/examples/kfserving/src/train.py b/docs/examples/kfserving/src/train.py deleted file mode 100644 index 152ba8ec..00000000 --- a/docs/examples/kfserving/src/train.py +++ /dev/null @@ -1,19 +0,0 @@ -import joblib -from sklearn.linear_model import LogisticRegression -from src.constants import SKLearnFolder, XGBFolder -from src.data import IrisData - - -def train_lr(artifacts_folder: str, data: IrisData): - logreg = LogisticRegression(C=1e5) - logreg.fit(data.X, data.y) - with open(f"{artifacts_folder}/{SKLearnFolder}/model.joblib", "wb") as f: - joblib.dump(logreg, f) - - -def train_xgb(artifacts_folder: str, data: IrisData): - from xgboost import XGBClassifier - - clf = XGBClassifier() - clf.fit(data.X, data.y) - clf.save_model(f"{artifacts_folder}/{XGBFolder}/model.bst") diff --git a/docs/examples/kfserving/tests/test_deploy.py b/docs/examples/kfserving/tests/test_deploy.py deleted file mode 100644 index 74046f62..00000000 --- a/docs/examples/kfserving/tests/test_deploy.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy as np -from src.constants import SKLearnTag, XGBoostTag -from src.tempo import get_tempo_artifacts - - -def test_sklearn_model_used(): - classifier, _, _ = get_tempo_artifacts("") - classifier.models.sklearn = lambda input: np.array([[1]]) - res, tag = classifier(np.array([[1, 2, 3, 4]])) - assert res[0][0] == 1 - assert tag == SKLearnTag - - -def test_xgboost_model_used(): - classifier, _, _ = get_tempo_artifacts("") - classifier.models.sklearn = lambda input: np.array([[0.2]]) - classifier.models.xgboost = lambda input: np.array([[0.1]]) - res, tag = classifier(np.array([[1, 2, 3, 4]])) - assert res[0][0] == 0.1 - assert tag == XGBoostTag diff --git a/docs/examples/multi-model/README.ipynb b/docs/examples/multi-model/README.ipynb index e6a15779..5551e2d2 100644 --- a/docs/examples/multi-model/README.ipynb +++ b/docs/examples/multi-model/README.ipynb @@ -43,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 1, "id": "numeric-mouse", "metadata": { "scrolled": true @@ -88,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 2, "id": "stainless-reggae", "metadata": {}, "outputs": [], @@ -104,7 +104,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 3, "id": "careful-question", "metadata": { "code_folding": [ @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 4, "id": "southeast-heritage", "metadata": {}, "outputs": [ @@ -146,14 +146,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "[18:56:15] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" + "[16:24:59] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/home/clive/anaconda3/envs/tempo-dev/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n", + "/home/clive/anaconda3/envs/tempo-examples/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n", " warnings.warn(label_encoder_deprecation_msg, UserWarning)\n" ] } @@ -179,7 +179,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "id": "collectible-sample", "metadata": {}, "outputs": [], @@ -190,7 +190,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 6, "id": "secure-perth", "metadata": { "code_folding": [] @@ -261,7 +261,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "id": "comic-figure", "metadata": { "code_folding": [] @@ -301,14 +301,14 @@ "output_type": "stream", "text": [ "\u001b[1m============================= test session starts ==============================\u001b[0m\n", - "platform linux -- Python 3.7.10, pytest-6.2.0, py-1.10.0, pluggy-0.13.1\n", + "platform linux -- Python 3.7.9, pytest-6.2.0, py-1.10.0, pluggy-0.13.1\n", "rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg\n", - "plugins: cases-3.4.6, asyncio-0.14.0\n", + "plugins: cases-3.4.6, cov-2.12.1, asyncio-0.14.0\n", "collected 2 items \u001b[0m\u001b[1m\n", "\n", "tests/test_tempo.py \u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[32m [100%]\u001b[0m\n", "\n", - "\u001b[32m============================== \u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.40s\u001b[0m\u001b[32m ===============================\u001b[0m\n" + "\u001b[32m============================== \u001b[32m\u001b[1m2 passed\u001b[0m\u001b[32m in 1.17s\u001b[0m\u001b[32m ===============================\u001b[0m\n" ] } ], @@ -355,8 +355,8 @@ "output_type": "stream", "text": [ "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-2e9f7838-e194-4a05-ae8f-22a337724906' to '/home/clive/work/mlops/fork-tempo/docs/examples/multi-model/artifacts/classifier/environment.tar.gz'\n", - "[########################################] | 100% Completed | 12.1s\n" + "Packing environment at '/home/clive/anaconda3/envs/tempo-0b068b2d-6246-44e7-91cc-ea0c2e210e09' to '/home/clive/work/mlops/fork-tempo/docs/examples/multi-model/artifacts/classifier/environment.tar.gz'\n", + "[########################################] | 100% Completed | 16.2s\n" ] } ], @@ -442,10 +442,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "secret/minio-secret created\r\n", - "serviceaccount/tempo-pipeline created\r\n", - "role.rbac.authorization.k8s.io/tempo-pipeline created\r\n", - "rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding created\r\n" + "secret/minio-secret configured\r\n", + "serviceaccount/tempo-pipeline unchanged\r\n", + "role.rbac.authorization.k8s.io/tempo-pipeline unchanged\r\n", + "rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding unchanged\r\n" ] } ], @@ -480,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 17, "id": "civil-arctic", "metadata": {}, "outputs": [], @@ -496,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 18, "id": "binary-denial", "metadata": {}, "outputs": [], @@ -507,7 +507,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 19, "id": "psychological-clerk", "metadata": {}, "outputs": [ @@ -515,7 +515,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'}\n", + "{'output0': array([1]), 'output1': 'sklearn prediction'}\n", "{'output0': array([[0.00847207, 0.03168793, 0.95984 ]], dtype=float32), 'output1': 'xgboost prediction'}\n" ] } @@ -535,7 +535,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 20, "id": "invalid-funds", "metadata": {}, "outputs": [ @@ -562,7 +562,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 21, "id": "lesser-convertible", "metadata": {}, "outputs": [ @@ -573,7 +573,7 @@ " 'output1': 'xgboost prediction'}" ] }, - "execution_count": 23, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } @@ -584,7 +584,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 22, "id": "blind-flower", "metadata": {}, "outputs": [], @@ -605,7 +605,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 23, "id": "female-graduation", "metadata": {}, "outputs": [], @@ -625,7 +625,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 24, "id": "eight-expansion", "metadata": {}, "outputs": [ @@ -645,13 +645,13 @@ " [{\"ty\": \"numpy.ndarray\", \"name\": \"payload\"}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", " \"name\": null}, {\"ty\": \"builtins.str\", \"name\": null}]}, \"description\": \"A pipeline\r\n", " to use either an sklearn or xgboost model for Iris classification\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.seldon.SeldonKubernetesRuntime\", \"state_options\": {\"state_type\": \"LOCAL\",\r\n", - " \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", + " \"tempo.protocols.v2.V2Protocol\", \"runtime_options\": {\"runtime\": \"tempo.seldon.SeldonKubernetesRuntime\",\r\n", + " \"state_options\": {\"state_type\": \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\":\r\n", + " \"\"}, \"insights_options\": {\"worker_endpoint\": \"\", \"batch_size\": 1, \"parallelism\":\r\n", + " 1, \"retries\": 3, \"window_time\": 0, \"mode_type\": \"NONE\", \"in_asyncio\": false},\r\n", + " \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\", \"ssl\": false,\r\n", + " \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\": null,\r\n", + " \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", " false, \"namespace\": \"production\"}}'\r\n", " labels:\r\n", " seldon.io/tempo: \"true\"\r\n", @@ -693,13 +693,13 @@ " \"uri\": \"s3://tempo/basic/sklearn\", \"platform\": \"sklearn\", \"inputs\": {\"args\":\r\n", " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", " \"name\": null}]}, \"description\": \"An SKLearn Iris classification model\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.seldon.SeldonKubernetesRuntime\", \"state_options\": {\"state_type\": \"LOCAL\",\r\n", - " \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", + " \"tempo.protocols.v2.V2Protocol\", \"runtime_options\": {\"runtime\": \"tempo.seldon.SeldonKubernetesRuntime\",\r\n", + " \"state_options\": {\"state_type\": \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\":\r\n", + " \"\"}, \"insights_options\": {\"worker_endpoint\": \"\", \"batch_size\": 1, \"parallelism\":\r\n", + " 1, \"retries\": 3, \"window_time\": 0, \"mode_type\": \"NONE\", \"in_asyncio\": false},\r\n", + " \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\", \"ssl\": false,\r\n", + " \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\": null,\r\n", + " \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", " false, \"namespace\": \"production\"}}'\r\n", " labels:\r\n", " seldon.io/tempo: \"true\"\r\n", @@ -729,13 +729,13 @@ " \"uri\": \"s3://tempo/basic/xgboost\", \"platform\": \"xgboost\", \"inputs\": {\"args\":\r\n", " [{\"ty\": \"numpy.ndarray\", \"name\": null}]}, \"outputs\": {\"args\": [{\"ty\": \"numpy.ndarray\",\r\n", " \"name\": null}]}, \"description\": \"An XGBoost Iris classification model\"}, \"protocol\":\r\n", - " \"tempo.kfserving.protocol.KFServingV2Protocol\", \"runtime_options\": {\"runtime\":\r\n", - " \"tempo.seldon.SeldonKubernetesRuntime\", \"state_options\": {\"state_type\": \"LOCAL\",\r\n", - " \"key_prefix\": \"\", \"host\": \"\", \"port\": \"\"}, \"insights_options\": {\"worker_endpoint\":\r\n", - " \"\", \"batch_size\": 1, \"parallelism\": 1, \"retries\": 3, \"window_time\": 0, \"mode_type\":\r\n", - " \"NONE\", \"in_asyncio\": false}, \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\",\r\n", - " \"ssl\": false, \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\":\r\n", - " null, \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", + " \"tempo.protocols.v2.V2Protocol\", \"runtime_options\": {\"runtime\": \"tempo.seldon.SeldonKubernetesRuntime\",\r\n", + " \"state_options\": {\"state_type\": \"LOCAL\", \"key_prefix\": \"\", \"host\": \"\", \"port\":\r\n", + " \"\"}, \"insights_options\": {\"worker_endpoint\": \"\", \"batch_size\": 1, \"parallelism\":\r\n", + " 1, \"retries\": 3, \"window_time\": 0, \"mode_type\": \"NONE\", \"in_asyncio\": false},\r\n", + " \"ingress_options\": {\"ingress\": \"tempo.ingress.istio.IstioIngress\", \"ssl\": false,\r\n", + " \"verify_ssl\": true}, \"replicas\": 1, \"minReplicas\": null, \"maxReplicas\": null,\r\n", + " \"authSecretName\": \"minio-secret\", \"serviceAccountName\": null, \"add_svc_orchestrator\":\r\n", " false, \"namespace\": \"production\"}}'\r\n", " labels:\r\n", " seldon.io/tempo: \"true\"\r\n", @@ -786,7 +786,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.7.9" } }, "nbformat": 4, diff --git a/docs/examples/multi-model/README.md b/docs/examples/multi-model/README.md index 9fcc597f..db2915da 100644 --- a/docs/examples/multi-model/README.md +++ b/docs/examples/multi-model/README.md @@ -94,10 +94,10 @@ train_sklearn(data, ARTIFACTS_FOLDER) train_xgboost(data, ARTIFACTS_FOLDER) ``` - [18:56:15] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior. + [16:24:59] WARNING: ../src/learner.cc:1095: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'multi:softprob' was changed from 'merror' to 'mlogloss'. Explicitly set eval_metric if you'd like to restore the old behavior. - /home/clive/anaconda3/envs/tempo-dev/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1]. + /home/clive/anaconda3/envs/tempo-examples/lib/python3.7/site-packages/xgboost/sklearn.py:1146: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1]. warnings.warn(label_encoder_deprecation_msg, UserWarning) @@ -202,14 +202,14 @@ def test_xgboost_model_used(): ``` ============================= test session starts ============================== - platform linux -- Python 3.7.10, pytest-6.2.0, py-1.10.0, pluggy-0.13.1 + platform linux -- Python 3.7.9, pytest-6.2.0, py-1.10.0, pluggy-0.13.1 rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg - plugins: cases-3.4.6, asyncio-0.14.0 + plugins: cases-3.4.6, cov-2.12.1, asyncio-0.14.0 collected 2 items  tests/test_tempo.py .. [100%] - ============================== 2 passed in 1.40s =============================== + ============================== 2 passed in 1.17s =============================== ## Save Classifier Environment @@ -231,8 +231,8 @@ save(classifier) ``` Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-2e9f7838-e194-4a05-ae8f-22a337724906' to '/home/clive/work/mlops/fork-tempo/docs/examples/multi-model/artifacts/classifier/environment.tar.gz' - [########################################] | 100% Completed | 12.1s + Packing environment at '/home/clive/anaconda3/envs/tempo-0b068b2d-6246-44e7-91cc-ea0c2e210e09' to '/home/clive/work/mlops/fork-tempo/docs/examples/multi-model/artifacts/classifier/environment.tar.gz' + [########################################] | 100% Completed | 16.2s ## Test Locally on Docker @@ -276,10 +276,10 @@ Create a Kind Kubernetes cluster with Minio and Seldon Core installed using Ansi !kubectl apply -f k8s/rbac -n production ``` - secret/minio-secret created - serviceaccount/tempo-pipeline created - role.rbac.authorization.k8s.io/tempo-pipeline created - rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding created + secret/minio-secret configured + serviceaccount/tempo-pipeline unchanged + role.rbac.authorization.k8s.io/tempo-pipeline unchanged + rolebinding.rbac.authorization.k8s.io/tempo-pipeline-rolebinding unchanged @@ -320,7 +320,7 @@ print(remote_model.predict(payload=np.array([[0, 0, 0, 0]]))) print(remote_model.predict(payload=np.array([[1, 2, 3, 4]]))) ``` - {'output0': array([1.], dtype=float32), 'output1': 'sklearn prediction'} + {'output0': array([1]), 'output1': 'sklearn prediction'} {'output0': array([[0.00847207, 0.03168793, 0.95984 ]], dtype=float32), 'output1': 'xgboost prediction'} @@ -398,13 +398,13 @@ with open(os.getcwd()+"/k8s/tempo.yaml","w") as f: [{"ty": "numpy.ndarray", "name": "payload"}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}, {"ty": "builtins.str", "name": null}]}, "description": "A pipeline to use either an sklearn or xgboost model for Iris classification"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: "true" @@ -446,13 +446,13 @@ with open(os.getcwd()+"/k8s/tempo.yaml","w") as f: "uri": "s3://tempo/basic/sklearn", "platform": "sklearn", "inputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "description": "An SKLearn Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: "true" @@ -482,13 +482,13 @@ with open(os.getcwd()+"/k8s/tempo.yaml","w") as f: "uri": "s3://tempo/basic/xgboost", "platform": "xgboost", "inputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "description": "An XGBoost Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: "true" diff --git a/docs/examples/multi-model/k8s/tempo.yaml b/docs/examples/multi-model/k8s/tempo.yaml index 0388f9dd..a89b719c 100644 --- a/docs/examples/multi-model/k8s/tempo.yaml +++ b/docs/examples/multi-model/k8s/tempo.yaml @@ -10,13 +10,13 @@ metadata: [{"ty": "numpy.ndarray", "name": "payload"}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}, {"ty": "builtins.str", "name": null}]}, "description": "A pipeline to use either an sklearn or xgboost model for Iris classification"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: 'true' @@ -50,7 +50,7 @@ spec: true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}' - image: seldonio/mlserver:0.3.2 + image: seldonio/mlserver:0.4.1 name: classifier graph: envSecretRefName: minio-secret @@ -74,13 +74,13 @@ metadata: "uri": "s3://tempo/basic/sklearn", "platform": "sklearn", "inputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "description": "An SKLearn Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: 'true' @@ -111,13 +111,13 @@ metadata: "uri": "s3://tempo/basic/xgboost", "platform": "xgboost", "inputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "outputs": {"args": [{"ty": "numpy.ndarray", "name": null}]}, "description": "An XGBoost Iris classification model"}, "protocol": - "tempo.kfserving.protocol.KFServingV2Protocol", "runtime_options": {"runtime": - "tempo.seldon.SeldonKubernetesRuntime", "state_options": {"state_type": "LOCAL", - "key_prefix": "", "host": "", "port": ""}, "insights_options": {"worker_endpoint": - "", "batch_size": 1, "parallelism": 1, "retries": 3, "window_time": 0, "mode_type": - "NONE", "in_asyncio": false}, "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", - "ssl": false, "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": - null, "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": + "tempo.protocols.v2.V2Protocol", "runtime_options": {"runtime": "tempo.seldon.SeldonKubernetesRuntime", + "state_options": {"state_type": "LOCAL", "key_prefix": "", "host": "", "port": + ""}, "insights_options": {"worker_endpoint": "", "batch_size": 1, "parallelism": + 1, "retries": 3, "window_time": 0, "mode_type": "NONE", "in_asyncio": false}, + "ingress_options": {"ingress": "tempo.ingress.istio.IstioIngress", "ssl": false, + "verify_ssl": true}, "replicas": 1, "minReplicas": null, "maxReplicas": null, + "authSecretName": "minio-secret", "serviceAccountName": null, "add_svc_orchestrator": false, "namespace": "production"}}' labels: seldon.io/tempo: 'true' diff --git a/docs/examples/outlier/README.ipynb b/docs/examples/outlier/README.ipynb index 68c340a6..e908747a 100644 --- a/docs/examples/outlier/README.ipynb +++ b/docs/examples/outlier/README.ipynb @@ -204,12 +204,10 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "c0b0af26", "metadata": { - "code_folding": [ - 0 - ] + "code_folding": [] }, "outputs": [], "source": [ @@ -221,7 +219,8 @@ "from alibi_detect.base import NumpyEncoder\n", "from src.constants import ARTIFACTS_FOLDER, MODEL_FOLDER, OUTLIER_FOLDER\n", "\n", - "from tempo.kfserving.protocol import KFServingV1Protocol, KFServingV2Protocol\n", + "from tempo.protocols.v2 import V2Protocol\n", + "from tempo.protocols.tensorflow import TensorflowProtocol\n", "from tempo.serve.metadata import ModelFramework\n", "from tempo.serve.model import Model\n", "from tempo.serve.pipeline import PipelineModels\n", @@ -232,7 +231,7 @@ " @model(\n", " name=\"outlier\",\n", " platform=ModelFramework.Custom,\n", - " protocol=KFServingV2Protocol(),\n", + " protocol=V2Protocol(),\n", " uri=\"s3://tempo/outlier/cifar10/outlier\",\n", " local_folder=os.path.join(ARTIFACTS_FOLDER, OUTLIER_FOLDER),\n", " )\n", @@ -264,7 +263,7 @@ "\n", " cifar10_model = Model(\n", " name=\"resnet32\",\n", - " protocol=KFServingV1Protocol(),\n", + " protocol=TensorflowProtocol(),\n", " platform=ModelFramework.Tensorflow,\n", " uri=\"gs://seldon-models/tfserving/cifar10/resnet32\",\n", " local_folder=os.path.join(ARTIFACTS_FOLDER, MODEL_FOLDER),\n", @@ -276,7 +275,7 @@ "def create_svc_cls(outlier, model):\n", " @pipeline(\n", " name=\"cifar10-service\",\n", - " protocol=KFServingV2Protocol(),\n", + " protocol=V2Protocol(),\n", " uri=\"s3://tempo/outlier/cifar10/svc\",\n", " local_folder=os.path.join(ARTIFACTS_FOLDER, \"svc\"),\n", " models=PipelineModels(outlier=outlier, cifar10=model),\n", @@ -305,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "id": "8159cbec", "metadata": { "code_folding": [ @@ -345,7 +344,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "id": "aa78ec19", "metadata": {}, "outputs": [ @@ -356,7 +355,7 @@ "\u001b[1m============================= test session starts ==============================\u001b[0m\n", "platform linux -- Python 3.7.9, pytest-6.2.0, py-1.10.0, pluggy-0.13.1\n", "rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg\n", - "plugins: cases-3.4.6, asyncio-0.14.0\n", + "plugins: cases-3.4.6, cov-2.12.1, asyncio-0.14.0\n", "collected 2 items \u001b[0m\u001b[1m\n", "\n", "tests/test_tempo.py \u001b[32m.\u001b[0m\u001b[32m.\u001b[0m\u001b[33m [100%]\u001b[0m\n", @@ -371,7 +370,7 @@ " DeprecationWarning,\n", "\n", "-- Docs: https://docs.pytest.org/en/stable/warnings.html\n", - "\u001b[33m======================== \u001b[32m2 passed\u001b[0m, \u001b[33m\u001b[1m2 warnings\u001b[0m\u001b[33m in 3.77s\u001b[0m\u001b[33m =========================\u001b[0m\n", + "\u001b[33m======================== \u001b[32m2 passed\u001b[0m, \u001b[33m\u001b[1m2 warnings\u001b[0m\u001b[33m in 4.69s\u001b[0m\u001b[33m =========================\u001b[0m\n", "Unresolved object in checkpoint: (root).encoder.fc_mean.kernel\n", "Unresolved object in checkpoint: (root).encoder.fc_mean.bias\n", "Unresolved object in checkpoint: (root).encoder.fc_log_var.kernel\n", @@ -399,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "id": "61e4d540", "metadata": {}, "outputs": [ @@ -408,8 +407,8 @@ "output_type": "stream", "text": [ "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-c08c4322-62be-4461-82bc-d69ae2432671' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/outlier/environment.tar.gz'\n", - "[########################################] | 100% Completed | 1min 9.5s\n" + "Packing environment at '/home/clive/anaconda3/envs/tempo-c4fe11aa-1cd6-43dd-9fab-0dcb4fca7a62' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/outlier/environment.tar.gz'\n", + "[########################################] | 100% Completed | 1min 21.6s\n" ] } ], @@ -419,7 +418,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "id": "3c23ab3b", "metadata": {}, "outputs": [ @@ -428,8 +427,8 @@ "output_type": "stream", "text": [ "Collecting packages...\n", - "Packing environment at '/home/clive/anaconda3/envs/tempo-27f221b3-8635-4b7e-ace6-443f6d7e3b15' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/svc/environment.tar.gz'\n", - "[########################################] | 100% Completed | 11.5s\n" + "Packing environment at '/home/clive/anaconda3/envs/tempo-cfface3b-1080-47d2-a3b6-113db8e286e5' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/svc/environment.tar.gz'\n", + "[########################################] | 100% Completed | 16.1s\n" ] } ], @@ -449,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "747d2d82", "metadata": {}, "outputs": [], @@ -460,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "id": "b22014e7", "metadata": {}, "outputs": [ @@ -484,7 +483,7 @@ " 6.90874735e-09, 1.07275586e-11]])" ] }, - "execution_count": 14, + "execution_count": 15, "metadata": {}, "output_type": "execute_result" } @@ -497,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "id": "03ef662f", "metadata": {}, "outputs": [ @@ -519,7 +518,7 @@ "array([], dtype=float64)" ] }, - "execution_count": 15, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -534,7 +533,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "id": "6c6ea7c7", "metadata": {}, "outputs": [], diff --git a/docs/examples/outlier/README.md b/docs/examples/outlier/README.md index 5a1487a9..456bdacd 100644 --- a/docs/examples/outlier/README.md +++ b/docs/examples/outlier/README.md @@ -122,7 +122,8 @@ import numpy as np from alibi_detect.base import NumpyEncoder from src.constants import ARTIFACTS_FOLDER, MODEL_FOLDER, OUTLIER_FOLDER -from tempo.kfserving.protocol import KFServingV1Protocol, KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol +from tempo.protocols.tensorflow import TensorflowProtocol from tempo.serve.metadata import ModelFramework from tempo.serve.model import Model from tempo.serve.pipeline import PipelineModels @@ -133,7 +134,7 @@ def create_outlier_cls(): @model( name="outlier", platform=ModelFramework.Custom, - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), uri="s3://tempo/outlier/cifar10/outlier", local_folder=os.path.join(ARTIFACTS_FOLDER, OUTLIER_FOLDER), ) @@ -165,7 +166,7 @@ def create_model(): cifar10_model = Model( name="resnet32", - protocol=KFServingV1Protocol(), + protocol=TensorflowProtocol(), platform=ModelFramework.Tensorflow, uri="gs://seldon-models/tfserving/cifar10/resnet32", local_folder=os.path.join(ARTIFACTS_FOLDER, MODEL_FOLDER), @@ -177,7 +178,7 @@ def create_model(): def create_svc_cls(outlier, model): @pipeline( name="cifar10-service", - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), uri="s3://tempo/outlier/cifar10/svc", local_folder=os.path.join(ARTIFACTS_FOLDER, "svc"), models=PipelineModels(outlier=outlier, cifar10=model), @@ -239,7 +240,7 @@ def test_svc_inlier(): ============================= test session starts ============================== platform linux -- Python 3.7.9, pytest-6.2.0, py-1.10.0, pluggy-0.13.1 rootdir: /home/clive/work/mlops/fork-tempo, configfile: setup.cfg - plugins: cases-3.4.6, asyncio-0.14.0 + plugins: cases-3.4.6, cov-2.12.1, asyncio-0.14.0 collected 2 items  tests/test_tempo.py .. [100%] @@ -254,7 +255,7 @@ def test_svc_inlier(): DeprecationWarning, -- Docs: https://docs.pytest.org/en/stable/warnings.html - ======================== 2 passed, 2 warnings in 3.77s ========================= + ======================== 2 passed, 2 warnings in 4.69s ========================= Unresolved object in checkpoint: (root).encoder.fc_mean.kernel Unresolved object in checkpoint: (root).encoder.fc_mean.bias Unresolved object in checkpoint: (root).encoder.fc_log_var.kernel @@ -276,8 +277,8 @@ tempo.save(OutlierModel) ``` Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-c08c4322-62be-4461-82bc-d69ae2432671' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/outlier/environment.tar.gz' - [########################################] | 100% Completed | 1min 9.5s + Packing environment at '/home/clive/anaconda3/envs/tempo-c4fe11aa-1cd6-43dd-9fab-0dcb4fca7a62' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/outlier/environment.tar.gz' + [########################################] | 100% Completed | 1min 21.6s @@ -286,8 +287,8 @@ tempo.save(Cifar10Svc) ``` Collecting packages... - Packing environment at '/home/clive/anaconda3/envs/tempo-27f221b3-8635-4b7e-ace6-443f6d7e3b15' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/svc/environment.tar.gz' - [########################################] | 100% Completed | 11.5s + Packing environment at '/home/clive/anaconda3/envs/tempo-cfface3b-1080-47d2-a3b6-113db8e286e5' to '/home/clive/work/mlops/fork-tempo/docs/examples/outlier/artifacts/svc/environment.tar.gz' + [########################################] | 100% Completed | 16.1s ## Test Locally on Docker diff --git a/docs/examples/outlier/src/tempo.py b/docs/examples/outlier/src/tempo.py index 336094f0..e982d660 100644 --- a/docs/examples/outlier/src/tempo.py +++ b/docs/examples/outlier/src/tempo.py @@ -5,7 +5,8 @@ from alibi_detect.base import NumpyEncoder from src.constants import ARTIFACTS_FOLDER, MODEL_FOLDER, OUTLIER_FOLDER -from tempo.kfserving.protocol import KFServingV1Protocol, KFServingV2Protocol +from tempo.protocols.tensorflow import TensorflowProtocol +from tempo.protocols.v2 import V2Protocol from tempo.serve.metadata import ModelFramework from tempo.serve.model import Model from tempo.serve.pipeline import PipelineModels @@ -16,7 +17,7 @@ def create_outlier_cls(): @model( name="outlier", platform=ModelFramework.Custom, - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), uri="s3://tempo/outlier/cifar10/outlier", local_folder=os.path.join(ARTIFACTS_FOLDER, OUTLIER_FOLDER), ) @@ -48,7 +49,7 @@ def create_model(): cifar10_model = Model( name="resnet32", - protocol=KFServingV1Protocol(), + protocol=TensorflowProtocol(), platform=ModelFramework.Tensorflow, uri="gs://seldon-models/tfserving/cifar10/resnet32", local_folder=os.path.join(ARTIFACTS_FOLDER, MODEL_FOLDER), @@ -60,7 +61,7 @@ def create_model(): def create_svc_cls(outlier, model): @pipeline( name="cifar10-service", - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), uri="s3://tempo/outlier/cifar10/svc", local_folder=os.path.join(ARTIFACTS_FOLDER, "svc"), models=PipelineModels(outlier=outlier, cifar10=model), diff --git a/docs/index.rst b/docs/index.rst index aa7f867d..0d2f5e4e 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -22,7 +22,6 @@ Tempo Introductory Custom Python Model MLFlow end-to-end Example Multi-model Orchestration with Seldon Core - Multi-model Orchestration with KFServing Model Explanation (using Alibi) Outlier Detection (using Alibi-Detect) End to End Pipeline with Metaflow and Tempo diff --git a/docs/overview/architecture.md b/docs/overview/architecture.md index 6bbe0411..921fc805 100644 --- a/docs/overview/architecture.md +++ b/docs/overview/architecture.md @@ -37,11 +37,11 @@ Models when deployed will expose endpoints that respect a particular protocol. T | Protocol | Description | |--------|---------| -| KFServingV2Protocol | [V2 definition](https://github.com/kubeflow/kfserving/tree/master/docs/predict-api/v2) | -| KFServingV1Protocol | [Tensorflow protocol](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/api_rest.md) | +| V2Protocol | [V2 definition](https://github.com/kserve/kserve/tree/master/docs/predict-api/v2) | +| TensorflowProtocol | [Tensorflow protocol](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/api_rest.md) | | SeldonProtocol | [Seldon protcol definition](https://docs.seldon.io/projects/seldon-core/en/latest/graph/protocols.html#rest-and-grpc-seldon-protocol) | -The default protocol is KFServingV2 protocol. +The default protocol is the V2 protocol. If calling the model with tempo you will not need to deal with the protocol explicitly as translation from the defined python types to runtime payloads and vice versa will be automatic. diff --git a/docs/overview/quickstart.md b/docs/overview/quickstart.md index cd7cf460..39b3eb4e 100644 --- a/docs/overview/quickstart.md +++ b/docs/overview/quickstart.md @@ -44,15 +44,6 @@ cd ansible ansible-playbook playbooks/seldon_core.yaml ``` - -### Kubernetes Cluster with KFServing - -To create a Kind cluster with istio, knative serving, KFServing and Minio run: -```bash -cd ansible -ansible-playbook playbooks/kfserving.yaml -``` - ## Next Step Create the `tempo-examples` conda environment and try the [introductory example](../examples/custom-model/README.html) diff --git a/docs/overview/runtimes.md b/docs/overview/runtimes.md index 3b29357a..0a1c0625 100644 --- a/docs/overview/runtimes.md +++ b/docs/overview/runtimes.md @@ -17,5 +17,4 @@ The Runtimes defined within Tempo are: | ------- | --------------------- | -------- | | SeldonDockerRuntime | deploy Tempo models to Docker | [Custom model](../examples/custom-model/README.html) | | SeldonKubernetesRuntime | deploy Tempo models to a Kubernetes cluster with Seldon Core installed | [Multi-model](../examples/multi-model/README.html) | -| KFServingKubernetesRuntime | deploy Tempo models to a Kubernetes cluster with KFServing installed | [KFServing](../examples/kfserving/README.html) | | SeldonDeployRuntime | deploy Tempo models to a Kubernetes cluster with Seldon Deploy installed | | diff --git a/docs/workflow/workflow.md b/docs/workflow/workflow.md index 85740018..3355e54e 100644 --- a/docs/workflow/workflow.md +++ b/docs/workflow/workflow.md @@ -205,7 +205,7 @@ Once uploaded you can run your pipelines you can deploy to production in two mai #### Update RuntimeOptions with the production runtime -For Kubernetes you can use a Kubernetes Runtime such as [SeldonKubernetesRuntime](../api/tempo.seldon.k8s.html) or [KFServingKubernetesRuntime](../api/tempo.kfserving.k8s.html). +For Kubernetes you can use a Kubernetes Runtime such as [SeldonKubernetesRuntime](../api/tempo.seldon.k8s.html). Create appropriate Kubernetes settings as shown below for your use case. This may require creating the appropriate RBAC to allow components to access the remote bucket storage. diff --git a/tempo/aio/utils.py b/tempo/aio/utils.py index 74c1e54a..231b3653 100644 --- a/tempo/aio/utils.py +++ b/tempo/aio/utils.py @@ -1,6 +1,7 @@ from inspect import isclass -from ..kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol + from ..serve.metadata import BaseRuntimeOptionsType, DockerOptions, ModelFramework from ..serve.pipeline import PipelineModels from ..serve.protocol import Protocol @@ -12,7 +13,7 @@ def pipeline( name: str, - protocol: Protocol = KFServingV2Protocol(), + protocol: Protocol = V2Protocol(), local_folder: str = None, uri: str = None, models: PipelineModels = None, @@ -100,7 +101,7 @@ def model( inputs: ModelDataType = None, outputs: ModelDataType = None, conda_env: str = None, - protocol: Protocol = KFServingV2Protocol(), + protocol: Protocol = V2Protocol(), runtime_options: BaseRuntimeOptionsType = DockerOptions(), description: str = "", ): diff --git a/tempo/kfserving/__init__.py b/tempo/kfserving/__init__.py deleted file mode 100644 index dded7193..00000000 --- a/tempo/kfserving/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .k8s import KFServingKubernetesRuntime -from .protocol import KFServingV1Protocol, KFServingV2Protocol - -__all__ = ["KFServingV1Protocol", "KFServingV2Protocol", "KFServingKubernetesRuntime"] diff --git a/tempo/kfserving/endpoint.py b/tempo/kfserving/endpoint.py deleted file mode 100644 index 11cf8d91..00000000 --- a/tempo/kfserving/endpoint.py +++ /dev/null @@ -1,73 +0,0 @@ -import os -from urllib.parse import urlparse - -from kubernetes import client, config - -from tempo.seldon.specs import DefaultServiceAccountName -from tempo.serve.base import ModelSpec -from tempo.serve.ingress import create_ingress -from tempo.utils import logger - -ENV_K8S_SERVICE_HOST = "KUBERNETES_SERVICE_HOST" -ISTIO_GATEWAY = "istio" - - -class Endpoint(object): - """A Model Endpoint""" - - def __init__(self): - self.inside_cluster = os.getenv(ENV_K8S_SERVICE_HOST) - try: - if self.inside_cluster: - logger.debug("Loading cluster local config") - config.load_incluster_config() - else: - logger.debug("Loading external kubernetes config") - config.load_kube_config() - except Exception: - logger.warning("Failed to load kubeconfig. Only local mode is possible.") - - def get_service_host(self, model_spec: ModelSpec) -> str: - if self.inside_cluster is not None: - config.load_incluster_config() - api_instance = client.CustomObjectsApi() - api_response = api_instance.get_namespaced_custom_object_status( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - ) - if self.inside_cluster is None: - url = api_response["status"]["url"] - o = urlparse(url) - return o.hostname - else: - url = api_response["status"]["address"]["url"] - o = urlparse(url) - return o.hostname - - def get_url(self, model_spec: ModelSpec): - if self.inside_cluster is None: - ingress = create_ingress(model_spec) - ingress_host_url = ingress.get_external_host_url(model_spec) - return f"{ingress_host_url}" + model_spec.protocol.get_predict_path(model_spec.model_details) - else: - # TODO check why needed this here - config.load_incluster_config() - api_instance = client.CustomObjectsApi() - api_response = api_instance.get_namespaced_custom_object_status( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - ) - url = api_response["status"]["address"]["url"] - # Hack until https://github.com/kubeflow/kfserving/issues/1483 fixed - if ( - "serviceAccountName" in api_response["spec"]["predictor"] - and api_response["spec"]["predictor"]["serviceAccountName"] == DefaultServiceAccountName - ): - url = url.replace("v1/models", "v2/models").replace(":predict", "/infer") - return url diff --git a/tempo/kfserving/k8s.py b/tempo/kfserving/k8s.py deleted file mode 100644 index 5e09aba6..00000000 --- a/tempo/kfserving/k8s.py +++ /dev/null @@ -1,278 +0,0 @@ -import json -import os -import time -from typing import Dict, Optional, Sequence - -import yaml -from kubernetes import client, config -from kubernetes.client.rest import ApiException - -from tempo.k8s.constants import TempoK8sDescriptionAnnotation, TempoK8sLabel, TempoK8sModelSpecAnnotation -from tempo.kfserving.endpoint import Endpoint -from tempo.kfserving.protocol import KFServingV2Protocol -from tempo.seldon.constants import MLSERVER_IMAGE -from tempo.seldon.specs import DefaultModelsPath, DefaultServiceAccountName -from tempo.serve.base import ClientModel, ModelSpec, Runtime -from tempo.serve.constants import ENV_TEMPO_RUNTIME_OPTIONS -from tempo.serve.metadata import KubernetesRuntimeOptions, ModelFramework -from tempo.serve.stub import deserialize -from tempo.utils import logger - -DefaultHTTPPort = "8080" -DefaultGRPCPort = "9000" - -ENV_K8S_SERVICE_HOST = "KUBERNETES_SERVICE_HOST" - -Implementations = { - ModelFramework.SKLearn: "sklearn", - ModelFramework.XGBoost: "xgboost", - ModelFramework.Tensorflow: "tensorflow", - ModelFramework.PyTorch: "triton", - ModelFramework.ONNX: "triton", - ModelFramework.TensorRT: "triton", -} - - -class KFServingKubernetesRuntime(Runtime): - def __init__(self, runtime_options: Optional[KubernetesRuntimeOptions] = None): - if runtime_options is None: - runtime_options = KubernetesRuntimeOptions() - runtime_options.runtime = "tempo.kfserving.KFServingKubernetesRuntime" - super().__init__(runtime_options) - - def _inside_cluster(self): - return os.getenv(ENV_K8S_SERVICE_HOST) - - def create_k8s_client(self): - if self._inside_cluster(): - logger.debug("Loading cluster local config") - config.load_incluster_config() - return True - else: - logger.debug("Loading external kubernetes config") - config.load_kube_config() - return False - - def get_endpoint_spec(self, model_spec: ModelSpec) -> str: - endpoint = Endpoint() - return endpoint.get_url(model_spec) - - def get_headers(self, model_spec: ModelSpec) -> Dict[str, str]: - if not self._inside_cluster(): - endpoint = Endpoint() - service_host = endpoint.get_service_host(model_spec) - return {"Host": service_host} - else: - return {} - - def undeploy_spec(self, model_spec: ModelSpec): - self.create_k8s_client() - api_instance = client.CustomObjectsApi() - api_instance.delete_namespaced_custom_object( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - body=client.V1DeleteOptions(propagation_policy="Foreground"), - ) - - def deploy_spec(self, model_spec: ModelSpec): - spec = self._get_spec(model_spec) - logger.debug(model_spec) - self.create_k8s_client() - api_instance = client.CustomObjectsApi() - - try: - existing = api_instance.get_namespaced_custom_object( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - ) - spec["metadata"]["resourceVersion"] = existing["metadata"]["resourceVersion"] - api_instance.replace_namespaced_custom_object( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - spec, - ) - except ApiException as e: - if e.status == 404: - api_instance.create_namespaced_custom_object( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - spec, - ) - else: - raise e - - def wait_ready_spec(self, model_spec: ModelSpec, timeout_secs=None) -> bool: - self.create_k8s_client() - ready = False - t0 = time.time() - while not ready: - api_instance = client.CustomObjectsApi() - existing = api_instance.get_namespaced_custom_object( - "serving.kubeflow.org", - "v1beta1", - model_spec.runtime_options.namespace, # type: ignore - "inferenceservices", - model_spec.model_details.name, - ) - default_ready = False - routes_ready = False - conf_ready = False - ingress_ready = False - if "status" in existing and "conditions" in existing["status"]: - for item in existing["status"]["conditions"]: - if item["type"] == "PredictorReady": - default_ready = item["status"] == "True" - elif item["type"] == "PredictorRouteReady": - routes_ready = item["status"] == "True" - elif item["type"] == "IngressReady": - ingress_ready = item["status"] == "True" - elif item["type"] == "PredictorConfigurationReady": - conf_ready = item["status"] == "True" - ready = default_ready and routes_ready and ingress_ready and conf_ready - if timeout_secs is not None: - t1 = time.time() - if t1 - t0 > timeout_secs: - return ready - time.sleep(1) - return ready - - def _get_spec(self, model_spec: ModelSpec) -> dict: - if model_spec.model_details.platform == ModelFramework.TempoPipeline: - serviceAccountName = model_spec.runtime_options.serviceAccountName # type: ignore - if serviceAccountName is None: - serviceAccountName = DefaultServiceAccountName - return { - "apiVersion": "serving.kubeflow.org/v1beta1", - "kind": "InferenceService", - "metadata": { - "name": model_spec.model_details.name, - "namespace": model_spec.runtime_options.namespace, # type: ignore - "labels": { - TempoK8sLabel: "true", - }, - "annotations": { - TempoK8sDescriptionAnnotation: model_spec.model_details.description, - TempoK8sModelSpecAnnotation: model_spec.json(), - }, - }, - "spec": { - "predictor": { - "serviceAccountName": serviceAccountName, - "containers": [ - { - "image": MLSERVER_IMAGE, - "name": "mlserver", - "env": [ - { - "name": "STORAGE_URI", - "value": model_spec.model_details.uri, - }, - { - "name": "MLSERVER_HTTP_PORT", - "value": DefaultHTTPPort, - }, - { - "name": "MLSERVER_GRPC_PORT", - "value": DefaultGRPCPort, - }, - { - "name": "MLSERVER_MODEL_IMPLEMENTATION", - "value": "tempo.mlserver.InferenceRuntime", - }, - { - "name": "MLSERVER_MODEL_NAME", - "value": model_spec.model_details.name, - }, - { - "name": "MLSERVER_MODEL_URI", - "value": DefaultModelsPath, - }, - { - "name": ENV_TEMPO_RUNTIME_OPTIONS, - "value": json.dumps(model_spec.runtime_options.dict()), - }, - ], - }, - ], - }, - }, - } - elif model_spec.model_details.platform in Implementations: - model_implementation = Implementations[model_spec.model_details.platform] - spec: Dict = { - "apiVersion": "serving.kubeflow.org/v1beta1", - "kind": "InferenceService", - "metadata": { - "name": model_spec.model_details.name, - "namespace": model_spec.runtime_options.namespace, # type: ignore - "labels": { - TempoK8sLabel: "true", - }, - "annotations": { - TempoK8sDescriptionAnnotation: model_spec.model_details.description, - TempoK8sModelSpecAnnotation: model_spec.json(), - }, - }, - "spec": { - "predictor": { - model_implementation: {"storageUri": model_spec.model_details.uri}, - }, - }, - } - if model_spec.runtime_options.serviceAccountName is not None: # type: ignore - spec["spec"]["predictor"][ - "serviceAccountName" - ] = model_spec.runtime_options.serviceAccountName # type: ignore - if isinstance(model_spec.protocol, KFServingV2Protocol): - spec["spec"]["predictor"][model_implementation]["protocolVersion"] = "v2" - return spec - else: - raise ValueError( - "Can't create spec for implementation ", - model_spec.model_details.platform, - ) - - def to_k8s_yaml_spec(self, model_spec: ModelSpec) -> str: - d = self._get_spec(model_spec) - return yaml.safe_dump(d) - - def list_models(self, namespace: Optional[str] = None) -> Sequence[ClientModel]: - self.create_k8s_client() - api_instance = client.CustomObjectsApi() - - if namespace is None and self.runtime_options is not None: - namespace = self.runtime_options.namespace # type: ignore - - if namespace is None: - return [] - - try: - models = [] - response = api_instance.list_namespaced_custom_object( - group="serving.kubeflow.org", - version="v1beta1", - namespace=namespace, - plural="inferenceservices", - label_selector=TempoK8sLabel + "=true", - ) - for model in response["items"]: - metadata = model["metadata"]["annotations"][TempoK8sModelSpecAnnotation] - remote_model = deserialize(json.loads(metadata)) - models.append(remote_model) - return models - except ApiException as e: - if e.status == 404: - return [] - else: - raise e diff --git a/docs/examples/kfserving/__init__.py b/tempo/protocols/__init__.py similarity index 100% rename from docs/examples/kfserving/__init__.py rename to tempo/protocols/__init__.py diff --git a/tempo/seldon/protocol.py b/tempo/protocols/seldon.py similarity index 100% rename from tempo/seldon/protocol.py rename to tempo/protocols/seldon.py diff --git a/tempo/protocols/tensorflow.py b/tempo/protocols/tensorflow.py new file mode 100644 index 00000000..9a262e7b --- /dev/null +++ b/tempo/protocols/tensorflow.py @@ -0,0 +1,123 @@ +from typing import Any, Dict, List, Optional, Type, Union + +import numpy as np + +from tempo.protocols.v2 import V2Protocol +from tempo.serve.metadata import ModelDataArgs, ModelDetails +from tempo.serve.protocol import Protocol + + +class TensorflowProtocol(Protocol): + @staticmethod + def create_v1_from_np(arr: np.ndarray, name: str = None) -> list: + return arr.tolist() + + @staticmethod + def create_np_from_v1(data: list) -> np.ndarray: + arr = np.array(data) + return arr + + def get_predict_path(self, model_details: ModelDetails): + return f"/v1/models/{model_details.name}:predict" + + def get_status_path(self, model_details: ModelDetails) -> str: + return f"/v1/models/{model_details.name}" + + def to_protocol_request(self, *args, **kwargs) -> Dict: + if len(args) > 0 and len(kwargs.values()) > 0: + raise ValueError("KFserving V1 protocol only supports either named or unamed arguments but not both") + + inputs = [] + if len(args) > 0: + for raw in args: + raw_type = type(raw) + + if raw_type == np.ndarray: + inputs.append(TensorflowProtocol.create_v1_from_np(raw)) + else: + for (name, raw) in kwargs.items(): + raw_type = type(raw) + + if raw_type == np.ndarray: + inputs.append(TensorflowProtocol.create_v1_from_np(raw, name)) + else: + raise ValueError(f"Unknown input type {raw_type}") + + if len(inputs) == 1: + return {"instances": inputs[0]} + else: + return {"instances": inputs} + + @staticmethod + def get_ty(name: Optional[str], idx: int, tys: ModelDataArgs) -> Type: + ty = None + if name is not None: + ty = tys[name] + if ty is None: + ty = tys[idx] + if ty is None: + return np.ndarray + return ty + + def to_protocol_response(self, model_details: ModelDetails, *args, **kwargs) -> Dict: + outputs: List[Union[Dict, List]] = [] + if len(args) > 0: + for idx, raw in enumerate(args): + raw_type = type(raw) + + if raw_type == np.ndarray: + outputs.append(TensorflowProtocol.create_v1_from_np(raw)) + else: + raise ValueError(f"Unknown input type {raw_type}") + else: + for name, raw in kwargs.items(): + raw_type = type(raw) + + if raw_type == np.ndarray: + data = raw.tolist() + outputs.append({name: data}) + else: + raise ValueError(f"Unknown input type {raw_type}") + return {"predictions": outputs} + + def from_protocol_request(self, res: Dict, tys: ModelDataArgs) -> Any: + inp = {} + for idx, input in enumerate(res["inputs"]): + ty = TensorflowProtocol.get_ty(input["name"], idx, tys) + + if ty == np.ndarray: + arr = V2Protocol.create_np_from_v2(input["data"], input["datatype"], input["shape"]) + inp[input["name"]] = arr + else: + raise ValueError(f"Unknown ty {ty} in conversion") + + if len(inp) == 1: + return list(inp.values())[0] + else: + return inp + + def from_protocol_response(self, res: Dict, tys: ModelDataArgs) -> Any: + if len(tys) <= 1: + ty = TensorflowProtocol.get_ty(None, 0, tys) + + if ty == np.ndarray: + return TensorflowProtocol.create_np_from_v1(res["predictions"]) + else: + raise ValueError(f"Unknown ty {ty} in conversion") + else: + out = [] + for idx, output in enumerate(res["predictions"]): + if type(output) == list: + for idx2, it in enumerate(output): + ty = TensorflowProtocol.get_ty(None, idx, tys) + + if ty == np.ndarray: + arr = TensorflowProtocol.create_np_from_v1(it) + out.append(arr) + else: + raise ValueError(f"Unknown ty {ty} in conversion") + + if len(out) == 1: + return out[0] + else: + return out diff --git a/tempo/kfserving/protocol.py b/tempo/protocols/v2.py similarity index 50% rename from tempo/kfserving/protocol.py rename to tempo/protocols/v2.py index d9ef2057..3ee46271 100644 --- a/tempo/kfserving/protocol.py +++ b/tempo/protocols/v2.py @@ -1,5 +1,5 @@ from ast import literal_eval -from typing import Any, Dict, List, Optional, Type, Union +from typing import Any, Dict, Optional, Type import numpy as np @@ -27,7 +27,7 @@ _nptymap[np.dtype("float32")] = "FP32" # Ensure correct mapping for ambiguous type -class KFServingV2Protocol(Protocol): +class V2Protocol(Protocol): @staticmethod def create_v2_from_np(arr: np.ndarray, name: str) -> Dict: if arr.dtype in _nptymap: @@ -95,18 +95,18 @@ def to_protocol_request(self, *args, **kwargs) -> Dict: args_num += 1 if raw_type == np.ndarray: numpy_args_num += 1 - inputs.append(KFServingV2Protocol.create_v2_from_np(raw, "input-" + str(idx))) + inputs.append(V2Protocol.create_v2_from_np(raw, "input-" + str(idx))) else: - inputs.append(KFServingV2Protocol.create_v2_from_any(raw, "input-" + str(idx))) + inputs.append(V2Protocol.create_v2_from_any(raw, "input-" + str(idx))) else: for (name, raw) in kwargs.items(): raw_type = type(raw) args_num += 1 if raw_type == np.ndarray: numpy_args_num += 1 - inputs.append(KFServingV2Protocol.create_v2_from_np(raw, name)) + inputs.append(V2Protocol.create_v2_from_np(raw, name)) else: - inputs.append(KFServingV2Protocol.create_v2_from_any(raw, name)) + inputs.append(V2Protocol.create_v2_from_any(raw, name)) request_ret = {"inputs": inputs} np_inference_request_enabled = {"parameters": _REQUEST_NUMPY_CONTENT_TYPE} @@ -130,9 +130,9 @@ def to_protocol_response(self, model_details: ModelDetails, *args, **kwargs) -> raw_type = type(raw) if raw_type == np.ndarray: - outputs.append(KFServingV2Protocol.create_v2_from_np(raw, "output" + str(idx))) + outputs.append(V2Protocol.create_v2_from_np(raw, "output" + str(idx))) else: - outputs.append(KFServingV2Protocol.create_v2_from_any(raw, "output" + str(idx))) + outputs.append(V2Protocol.create_v2_from_any(raw, "output" + str(idx))) for name, raw in kwargs.items(): raw_type = type(raw) @@ -141,18 +141,18 @@ def to_protocol_response(self, model_details: ModelDetails, *args, **kwargs) -> data = raw.flatten().tolist() outputs.append({"name": name, "datatype": "FP32", "shape": shape, "data": data}) else: - outputs.append(KFServingV2Protocol.create_v2_from_any(raw, name)) + outputs.append(V2Protocol.create_v2_from_any(raw, name)) return {"model_name": model_details.name, "outputs": outputs} def from_protocol_request(self, res: Dict, tys: ModelDataArgs) -> Any: inp = {} for idx, input in enumerate(res["inputs"]): - ty = KFServingV2Protocol.get_ty(input["name"], idx, tys) + ty = V2Protocol.get_ty(input["name"], idx, tys) if input["datatype"] == "BYTES": - inp[input["name"]] = KFServingV2Protocol.convert_from_bytes(input, ty) + inp[input["name"]] = V2Protocol.convert_from_bytes(input, ty) elif ty == np.ndarray: - arr = KFServingV2Protocol.create_np_from_v2(input["data"], input["datatype"], input["shape"]) + arr = V2Protocol.create_np_from_v2(input["data"], input["datatype"], input["shape"]) inp[input["name"]] = arr else: raise ValueError(f"Unknown ty {ty} in conversion") @@ -165,12 +165,12 @@ def from_protocol_request(self, res: Dict, tys: ModelDataArgs) -> Any: def from_protocol_response(self, res: Dict, tys: ModelDataArgs) -> Any: out = {} for idx, output in enumerate(res["outputs"]): - ty = KFServingV2Protocol.get_ty(output["name"], idx, tys) + ty = V2Protocol.get_ty(output["name"], idx, tys) if output["datatype"] == "BYTES": - out[output["name"]] = KFServingV2Protocol.convert_from_bytes(output, ty) + out[output["name"]] = V2Protocol.convert_from_bytes(output, ty) elif ty == np.ndarray: - arr = KFServingV2Protocol.create_np_from_v2(output["data"], output["datatype"], output["shape"]) + arr = V2Protocol.create_np_from_v2(output["data"], output["datatype"], output["shape"]) out[output["name"]] = arr else: raise ValueError(f"Unknown ty {ty} in conversion") @@ -179,119 +179,3 @@ def from_protocol_response(self, res: Dict, tys: ModelDataArgs) -> Any: return list(out.values())[0] else: return out - - -class KFServingV1Protocol(Protocol): - @staticmethod - def create_v1_from_np(arr: np.ndarray, name: str = None) -> list: - return arr.tolist() - - @staticmethod - def create_np_from_v1(data: list) -> np.ndarray: - arr = np.array(data) - return arr - - def get_predict_path(self, model_details: ModelDetails): - return f"/v1/models/{model_details.name}:predict" - - def get_status_path(self, model_details: ModelDetails) -> str: - return f"/v1/models/{model_details.name}" - - def to_protocol_request(self, *args, **kwargs) -> Dict: - if len(args) > 0 and len(kwargs.values()) > 0: - raise ValueError("KFserving V1 protocol only supports either named or unamed arguments but not both") - - inputs = [] - if len(args) > 0: - for raw in args: - raw_type = type(raw) - - if raw_type == np.ndarray: - inputs.append(KFServingV1Protocol.create_v1_from_np(raw)) - else: - for (name, raw) in kwargs.items(): - raw_type = type(raw) - - if raw_type == np.ndarray: - inputs.append(KFServingV1Protocol.create_v1_from_np(raw, name)) - else: - raise ValueError(f"Unknown input type {raw_type}") - - if len(inputs) == 1: - return {"instances": inputs[0]} - else: - return {"instances": inputs} - - @staticmethod - def get_ty(name: Optional[str], idx: int, tys: ModelDataArgs) -> Type: - ty = None - if name is not None: - ty = tys[name] - if ty is None: - ty = tys[idx] - if ty is None: - return np.ndarray - return ty - - def to_protocol_response(self, model_details: ModelDetails, *args, **kwargs) -> Dict: - outputs: List[Union[Dict, List]] = [] - if len(args) > 0: - for idx, raw in enumerate(args): - raw_type = type(raw) - - if raw_type == np.ndarray: - outputs.append(KFServingV1Protocol.create_v1_from_np(raw)) - else: - raise ValueError(f"Unknown input type {raw_type}") - else: - for name, raw in kwargs.items(): - raw_type = type(raw) - - if raw_type == np.ndarray: - data = raw.tolist() - outputs.append({name: data}) - else: - raise ValueError(f"Unknown input type {raw_type}") - return {"predictions": outputs} - - def from_protocol_request(self, res: Dict, tys: ModelDataArgs) -> Any: - inp = {} - for idx, input in enumerate(res["inputs"]): - ty = KFServingV1Protocol.get_ty(input["name"], idx, tys) - - if ty == np.ndarray: - arr = KFServingV2Protocol.create_np_from_v2(input["data"], input["datatype"], input["shape"]) - inp[input["name"]] = arr - else: - raise ValueError(f"Unknown ty {ty} in conversion") - - if len(inp) == 1: - return list(inp.values())[0] - else: - return inp - - def from_protocol_response(self, res: Dict, tys: ModelDataArgs) -> Any: - if len(tys) <= 1: - ty = KFServingV1Protocol.get_ty(None, 0, tys) - - if ty == np.ndarray: - return KFServingV1Protocol.create_np_from_v1(res["predictions"]) - else: - raise ValueError(f"Unknown ty {ty} in conversion") - else: - out = [] - for idx, output in enumerate(res["predictions"]): - if type(output) == list: - for idx2, it in enumerate(output): - ty = KFServingV1Protocol.get_ty(None, idx, tys) - - if ty == np.ndarray: - arr = KFServingV1Protocol.create_np_from_v1(it) - out.append(arr) - else: - raise ValueError(f"Unknown ty {ty} in conversion") - - if len(out) == 1: - return out[0] - else: - return out diff --git a/tempo/seldon/__init__.py b/tempo/seldon/__init__.py index e6aaf359..39ed399b 100644 --- a/tempo/seldon/__init__.py +++ b/tempo/seldon/__init__.py @@ -1,7 +1,8 @@ +from tempo.protocols.seldon import SeldonProtocol + from .deploy import SeldonDeployRuntime from .docker import SeldonDockerRuntime from .k8s import SeldonKubernetesRuntime -from .protocol import SeldonProtocol __all__ = [ "SeldonDockerRuntime", diff --git a/tempo/seldon/specs.py b/tempo/seldon/specs.py index eaf5e9ba..a2b13738 100644 --- a/tempo/seldon/specs.py +++ b/tempo/seldon/specs.py @@ -1,7 +1,8 @@ import json from tempo.k8s.constants import TempoK8sDescriptionAnnotation, TempoK8sLabel, TempoK8sModelSpecAnnotation -from tempo.kfserving.protocol import KFServingV1Protocol, KFServingV2Protocol +from tempo.protocols.tensorflow import TensorflowProtocol +from tempo.protocols.v2 import V2Protocol from tempo.seldon.constants import MLSERVER_IMAGE, TRITON_IMAGE from tempo.serve.base import ModelSpec from tempo.serve.constants import ENV_TEMPO_RUNTIME_OPTIONS @@ -24,7 +25,7 @@ def get_container_spec(model_details: ModelSpec) -> dict: ): return _V2ContainerFactory.get_container_spec(model_details.model_details, runtime_options) - if isinstance(model_details.protocol, KFServingV2Protocol): + if isinstance(model_details.protocol, V2Protocol): return _V2ContainerFactory.get_container_spec(model_details.model_details, runtime_options) return _V1ContainerFactory.get_container_spec(model_details.model_details) @@ -220,10 +221,10 @@ def _get_component_specs(self) -> list: ] def _get_spec_protocol(self) -> str: - if isinstance(self._details.protocol, KFServingV2Protocol): + if isinstance(self._details.protocol, V2Protocol): return "kfserving" - if isinstance(self._details.protocol, KFServingV1Protocol): + if isinstance(self._details.protocol, TensorflowProtocol): return "tensorflow" return "seldon" diff --git a/tempo/serve/base.py b/tempo/serve/base.py index 4f1d0d03..1d8dc885 100644 --- a/tempo/serve/base.py +++ b/tempo/serve/base.py @@ -250,6 +250,7 @@ def remote_with_client(self, model_spec: ModelSpec, client_details: ClientDetail response_raw.raise_for_status() response_json = response_raw.json() + logger.debug("Response raw %s", response_json) output_schema = model_spec.model_details.outputs return prot.from_protocol_response(response_json, output_schema) @@ -273,7 +274,9 @@ def remote_with_spec(self, model_spec: ModelSpec, *args, **kwargs): response_json = response_raw.json() output_schema = model_spec.model_details.outputs - return prot.from_protocol_response(response_json, output_schema) + res = prot.from_protocol_response(response_json, output_schema) + logger.debug("protocol decoded %s", res) + return res def wait_ready(self, runtime: Runtime, timeout_secs=None): return runtime.wait_ready_spec(self._get_model_spec(runtime), timeout_secs=timeout_secs) @@ -331,7 +334,7 @@ def predict(self, *args, **kwargs): if self.client_details is not None: return super().remote_with_client(self.model_spec, self.client_details, *args, **kwargs) else: - super().predict(*args, **kwargs) + return super().predict(*args, **kwargs) def deploy(self, runtime: Runtime): logger.warn("Remote model %s can't be deployed", self.model_spec.model_details.name) diff --git a/tempo/serve/model.py b/tempo/serve/model.py index 4670b898..8a71868e 100644 --- a/tempo/serve/model.py +++ b/tempo/serve/model.py @@ -1,6 +1,6 @@ from typing import Any, Callable -from tempo.kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol from tempo.serve.base import BaseModel from tempo.serve.metadata import BaseRuntimeOptionsType, DockerOptions, ModelFramework from tempo.serve.protocol import Protocol @@ -11,7 +11,7 @@ class Model(BaseModel): def __init__( self, name: str, - protocol: Protocol = KFServingV2Protocol(), + protocol: Protocol = V2Protocol(), local_folder: str = None, uri: str = None, platform: ModelFramework = None, diff --git a/tempo/serve/utils.py b/tempo/serve/utils.py index 975b0b67..b7cc1092 100644 --- a/tempo/serve/utils.py +++ b/tempo/serve/utils.py @@ -3,7 +3,8 @@ from types import SimpleNamespace from typing import Any, Callable, Optional, Type -from ..kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol + from .base import BaseModel from .metadata import BaseRuntimeOptionsType, DockerOptions, ModelFramework from .model import Model @@ -119,7 +120,7 @@ def _bind_tempo_interface(artifact: Any, model: BaseModel) -> Any: def pipeline( name: str, - protocol: Protocol = KFServingV2Protocol(), + protocol: Protocol = V2Protocol(), local_folder: str = None, uri: str = None, models: PipelineModels = None, @@ -207,7 +208,7 @@ def model( inputs: ModelDataType = None, outputs: ModelDataType = None, conda_env: str = None, - protocol: Protocol = KFServingV2Protocol(), + protocol: Protocol = V2Protocol(), runtime_options: BaseRuntimeOptionsType = DockerOptions(), description: str = "", ): diff --git a/tests/conftest.py b/tests/conftest.py index 83cb0036..41a32c87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,8 @@ import yaml from tempo import Model, ModelFramework, Pipeline, PipelineModels, model, pipeline, predictmethod -from tempo.kfserving import KFServingV1Protocol, KFServingV2Protocol +from tempo.protocols.tensorflow import TensorflowProtocol +from tempo.protocols.v2 import V2Protocol from tempo.seldon import SeldonProtocol from tempo.serve.constants import MLServerEnvDeps from tempo.serve.metadata import KubernetesRuntimeOptions @@ -75,7 +76,7 @@ def xgboost_model() -> Model: def custom_model() -> Model: @model( name="custom-model", - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), platform=ModelFramework.Custom, ) def _custom_model(payload: np.ndarray) -> np.ndarray: @@ -114,7 +115,7 @@ def cifar10_model() -> Model: platform=ModelFramework.Tensorflow, uri="gs://seldon-models/tfserving/cifar10/resnet32", local_folder=model_path, - protocol=KFServingV1Protocol(), + protocol=TensorflowProtocol(), ) diff --git a/tests/kfserving/__init__.py b/tests/kfserving/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/kfserving/test_k8s.py b/tests/kfserving/test_k8s.py deleted file mode 100644 index 17dc8721..00000000 --- a/tests/kfserving/test_k8s.py +++ /dev/null @@ -1,20 +0,0 @@ -import numpy as np -import pytest - -from tempo.kfserving.k8s import KFServingKubernetesRuntime -from tempo.serve.metadata import ModelFramework -from tempo.serve.model import Model - - -@pytest.mark.skip(reason="needs k8s cluster") -def test_kfserving(): - model = Model( - name="sklearn-iris2", - runtime=KFServingKubernetesRuntime(), - platform=ModelFramework.SKLearn, - uri="gs://kfserving-samples/models/sklearn/iris", - local_folder="sklearn/model", - ) - # model.deploy() - res = model(np.array([[1, 2, 3, 4]])) - print(res) diff --git a/docs/examples/kfserving/src/__init__.py b/tests/protocols/__init__.py similarity index 100% rename from docs/examples/kfserving/src/__init__.py rename to tests/protocols/__init__.py diff --git a/tests/kfserving/test_protocol.py b/tests/protocols/test_v2.py similarity index 84% rename from tests/kfserving/test_protocol.py rename to tests/protocols/test_v2.py index dd538b43..7eb1cb50 100644 --- a/tests/kfserving/test_protocol.py +++ b/tests/protocols/test_v2.py @@ -1,7 +1,7 @@ import numpy as np import pytest -from tempo.kfserving.protocol import _REQUEST_NUMPY_CONTENT_TYPE, KFServingV2Protocol +from tempo.protocols.v2 import _REQUEST_NUMPY_CONTENT_TYPE, V2Protocol from tempo.serve.metadata import ModelDataArg, ModelDataArgs @@ -10,7 +10,7 @@ [("abc", [97, 98, 99]), ({"a": 1}, [123, 39, 97, 39, 58, 32, 49, 125])], ) def test_v2_from_any(data, expected): - d = KFServingV2Protocol.create_v2_from_any(data, "a") + d = V2Protocol.create_v2_from_any(data, "a") assert d["name"] == "a" assert d["data"] == expected assert d["datatype"] == "BYTES" @@ -22,19 +22,19 @@ def test_v2_from_any(data, expected): ) def test_convert_from_bytes(data, ty, expected): output = {"data": data} - res = KFServingV2Protocol.convert_from_bytes(output, ty) + res = V2Protocol.convert_from_bytes(output, ty) assert res == expected def test_v2_from_protocol_response(): res = {"outputs": [{"name": "a", "data": [97, 98, 99], "datatype": "BYTES"}]} modelTyArgs = ModelDataArgs(args=[ModelDataArg(ty=str, name=None)]) - v2 = KFServingV2Protocol() + v2 = V2Protocol() res = v2.from_protocol_response(res, modelTyArgs) def test_v2_to_protocol_request_numpy(): - v2 = KFServingV2Protocol() + v2 = V2Protocol() data = np.random.randn(1, 28 * 28) request = v2.to_protocol_request(data) expected_request = { @@ -46,7 +46,7 @@ def test_v2_to_protocol_request_numpy(): def test_v2_to_protocol_request_other(): - v2 = KFServingV2Protocol() + v2 = V2Protocol() data = 1 request = v2.to_protocol_request(data) # we should not have the "parameters", mainly so that content_type= "np" is not present. diff --git a/tests/seldon/test_deploy.py b/tests/seldon/test_deploy.py index 0722a98c..c6ed270c 100644 --- a/tests/seldon/test_deploy.py +++ b/tests/seldon/test_deploy.py @@ -1,9 +1,9 @@ import numpy as np import pytest +from tempo.protocols.seldon import SeldonProtocol from tempo.seldon.deploy import SeldonDeployRuntime from tempo.seldon.k8s import SeldonKubernetesRuntime -from tempo.seldon.protocol import SeldonProtocol from tempo.serve.metadata import ( EnterpriseRuntimeAuthType, EnterpriseRuntimeOptions, diff --git a/tests/seldon/test_specs.py b/tests/seldon/test_specs.py index f8cb146e..d5c5accc 100644 --- a/tests/seldon/test_specs.py +++ b/tests/seldon/test_specs.py @@ -1,4 +1,4 @@ -from tempo.seldon.protocol import SeldonProtocol +from tempo.protocols.seldon import SeldonProtocol from tempo.seldon.specs import KubernetesSpec, get_container_spec from tempo.serve.base import ModelSpec from tempo.serve.metadata import KubernetesRuntimeOptions, ModelDataArgs, ModelDetails, ModelFramework diff --git a/tests/serve/test_model.py b/tests/serve/test_model.py index 472e93ca..57275f10 100644 --- a/tests/serve/test_model.py +++ b/tests/serve/test_model.py @@ -5,7 +5,7 @@ import pytest from tempo import deploy_local -from tempo.kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol from tempo.serve.metadata import ModelFramework from tempo.serve.model import Model from tempo.serve.utils import model, predictmethod @@ -47,7 +47,7 @@ def test_custom_model(v2_input, expected): @model( name="custom", - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), platform=ModelFramework.Custom, ) def custom_model(a: np.ndarray) -> np.ndarray: diff --git a/tests/serve/test_stub.py b/tests/serve/test_stub.py index 9bc4ea1d..53f23993 100644 --- a/tests/serve/test_stub.py +++ b/tests/serve/test_stub.py @@ -2,7 +2,7 @@ import numpy as np -from tempo.kfserving.protocol import KFServingV2Protocol +from tempo.protocols.v2 import V2Protocol from tempo.serve.base import ModelSpec from tempo.serve.metadata import KFServingOptions, ModelDataArg, ModelDataArgs, ModelDetails, ModelFramework @@ -39,11 +39,11 @@ def test_model_spec(): inputs=ModelDataArgs(args=[ModelDataArg(ty=str)]), outputs=ModelDataArgs(args=[]), ), - protocol=KFServingV2Protocol(), + protocol=V2Protocol(), runtime_options=KFServingOptions().local_options, ) s = ms.json() j = json.loads(s) ms2 = ModelSpec(**j) - assert isinstance(ms2.protocol, KFServingV2Protocol) + assert isinstance(ms2.protocol, V2Protocol) assert ms2.model_details.inputs.args[0].ty == str diff --git a/tests/serve/test_yaml.py b/tests/serve/test_yaml.py index 685fdcc3..4df06c37 100644 --- a/tests/serve/test_yaml.py +++ b/tests/serve/test_yaml.py @@ -1,8 +1,8 @@ import pytest import yaml +from tempo.protocols.seldon import SeldonProtocol from tempo.seldon.k8s import SeldonKubernetesRuntime -from tempo.seldon.protocol import SeldonProtocol from tempo.serve.metadata import KubernetesRuntimeOptions, ModelFramework from tempo.serve.model import Model