From 0f9a054df063319b7f48c5f54211901f50c8f24d Mon Sep 17 00:00:00 2001 From: "Trawinski, Dariusz" Date: Wed, 27 Mar 2019 15:52:16 +0100 Subject: [PATCH] readme updates (#26) --- Makefile | 6 ++--- README.md | 9 ++++++-- example_client/README.md | 2 ++ example_k8s/README.md | 23 ++++++++++++++++++- example_k8s/openvino_model_server_resnet.yaml | 5 ++++ setup.py | 2 +- 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 151e875b3c..e4668318f8 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ CONFIG := "$(CONFIG)" ML_DIR := "$(MK_DIR)" HTTP_PROXY := "$(http_proxy)" HTTPS_PROXY := "$(https_proxy)" -OVMS_VERSION := "0.3" +OVMS_VERSION := "0.5" .PHONY: default install uninstall requirements \ venv test unit_test coverage style dist clean \ @@ -86,7 +86,7 @@ docker_build_src_ubuntu: @echo "Building docker image" @echo OpenVINO Model Server version: $(OVMS_VERSION) > version @echo Git commit: `git rev-parse HEAD` >> version - @echo OpenVINO version: 2018_R4 src >> version + @echo OpenVINO version: 2018_R5 src >> version @echo docker build -f Dockerfile --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" -t ie-serving-py:latest . @docker build -f Dockerfile --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" -t ie-serving-py:latest . @@ -102,7 +102,7 @@ docker_build_src_intelpython: @echo "Building docker image" @echo OpenVINO Model Server version: $(OVMS_VERSION) > version @echo Git commit: `git rev-parse HEAD` >> version - @echo OpenVINO version: 2018_R4 src >> version + @echo OpenVINO version: 2018_R5 src >> version @echo docker build -f Dockerfile_intelpython --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" -t ie-serving-py:latest . @docker build -f Dockerfile_intelpython --build-arg http_proxy=$(HTTP_PROXY) --build-arg https_proxy="$(HTTPS_PROXY)" -t ie-serving-py:latest . diff --git a/README.md b/README.md index 3019fb9249..1484201e1e 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ Using FPGA (TBD) ## API documentation -OpenVINO™ model server API is documented in proto buffer files in [tensorflow_serving_api](ie_serving/tensorflow_serving_api/prediction_service.proto). +OpenVINO™ model server API is documented in proto buffer files in [tensorflow_serving_api](ie_serving/tensorflow_serving_api/prediction_service.proto). **Note:** The implementations for *Predict* and *GetModelMetadata* function calls are currently available. These are the most generic function calls and should address most of the usage scenarios. [predict Function Spec](ie_serving/tensorflow_serving_api/predict.proto) has two message definitions: *PredictRequest* and *PredictResponse*. @@ -59,6 +59,8 @@ Refer to the example client code to learn how to use this API and submit the req [Client Code Example](example_client) +[Jupyter notebook demo](example_k8s/OVMS_demo.ipynb) + ## Benchmarking results @@ -72,6 +74,9 @@ Refer to the example client code to learn how to use this API and submit the req [gRPC](https://grpc.io/) +[Inference at scale in Kubernetes](https://www.intel.ai/inference-at-scale-in-kubernetes) + +[OpenVINO Model Server boosts AI](https://www.intel.ai/openvino-model-server-boosts-ai-inference-operations/) ## Troubleshooting ### Server logging @@ -173,6 +178,6 @@ To run tests limited to models to locally downloaded models use command: ## Contact -Submit Github issue to ask question, submit a request or report a bug. +Submit Github issue to ask question, request a feature or report a bug. diff --git a/example_client/README.md b/example_client/README.md index b637b66c1f..402b68993e 100644 --- a/example_client/README.md +++ b/example_client/README.md @@ -206,3 +206,5 @@ Inputs metadata: Outputs metadata: Output name: prob; shape: [1, 1000]; dtype: DT_FLOAT ``` + +Refer also to the usage demo in the [jupyter notebook](../example_k8s/OVMS_demo.ipynb). diff --git a/example_k8s/README.md b/example_k8s/README.md index 10d78abcd2..fbab71b917 100644 --- a/example_k8s/README.md +++ b/example_k8s/README.md @@ -6,6 +6,12 @@ and ensures high availability. Below are described simple examples which are using NFS and GCS (Google Cloud Storage) and S3 as the storage for the models. +## Jupyter notebook demo + +OpenVINO Model Server deployment in Kubernetes including +the evaluation from gRPC client is demonstrated in the [jupyter notebook demo](OVMS_demo.ipynb). It enables serving of a pre-trained ResNet50 model quantized to INT8 precision. + + ## NFS server deployment There are many possible ways to arrange NFS storage for kubernetes pods. In this example it used a procedure described on @@ -119,6 +125,21 @@ kubectl apply -f openvino_model_server_s3_rc.yaml kubectl apply -f openvino_model_server_service.yaml ``` +## Readiness checks + +By default Kubernetes starts assinging requests to the service pods when they are in `Running` state. In some cases +when the models are stored remotely, more time is needed to download and import the model. + +To avoid risk of requests being passed to not fully initialized pods, it is recommended to add Kubernetes readiness checks: + +```yaml + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 10 +``` +The port number should match the one configured as the pod exposed port. ## Testing @@ -143,4 +164,4 @@ Top 1: redbone. Processing time: 76.49 ms; speed 13.07 fps Top 1: redbone. Processing time: 67.03 ms; speed 14.92 fps Top 1: French loaf. Processing time: 67.34 ms; speed 14.85 fps Top 1: brass, memorial tablet, plaque. Processing time: 68.64 ms; speed 14.57 fps -``` \ No newline at end of file +``` diff --git a/example_k8s/openvino_model_server_resnet.yaml b/example_k8s/openvino_model_server_resnet.yaml index f1b89f5d47..0ebc3ef794 100644 --- a/example_k8s/openvino_model_server_resnet.yaml +++ b/example_k8s/openvino_model_server_resnet.yaml @@ -20,6 +20,11 @@ spec: image: intelaipg/openvino-model-server:latest ports: - containerPort: 80 + readinessProbe: + tcpSocket: + port: 80 + initialDelaySeconds: 5 + periodSeconds: 10 env: - name: LOG_LEVEL value: "DEBUG" diff --git a/setup.py b/setup.py index 24e3974cdc..80421c7a58 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name='ie_serving', - version=0.4, + version=0.5, description="DLDT inference server", long_description="""DLDT inference server""", keywords='',