From 19cac7819d566e42aae5f99782d0638d187d6492 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Wed, 26 Apr 2023 00:40:04 -0700 Subject: [PATCH 01/13] load itex --- WORKSPACE | 7 +++--- tensorflow_serving/model_servers/BUILD | 22 ++++++++++------- tensorflow_serving/model_servers/main.cc | 29 +++++++++++++++++++++-- tensorflow_serving/model_servers/server.h | 1 + 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 8c9ff35dab0..1eb96757e55 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,11 +21,10 @@ local_repository( # and update the 'sha256' arg with the result. # 3. Request the new archive to be mirrored on mirror.bazel.build for more # reliable downloads. -load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") -tensorflow_http_archive( +#load("//tensorflow_serving:repo.bzl") +local_repository( name = "org_tensorflow", - sha256 = "4d4f15fa4e52f7c77a1793656f85d260225e2638aee34a9c7625583e0608265c", - git_commit = "332077006a5ed7004ee19fbae306768f8f500ca8", + path = "/home/gta/jianyi/tensorflow", ) # Import all of TensorFlow Serving's external dependencies. diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 2809e2afb73..dc7493b9b7a 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -382,8 +382,11 @@ cc_library( srcs = [ "server.cc", ], - hdrs = ["server.h"], + hdrs = ["server.h", + "version.h", + ], visibility = ["//visibility:public"], + linkstamp = "version.cc", deps = [ ":grpc_status_util", ":http_server", @@ -412,30 +415,28 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/c:c_api", + "@org_tensorflow//tensorflow/c:c_api_experimental", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core/profiler/rpc:profiler_service_impl", ] + SUPPORTED_TENSORFLOW_OPS, + alwayslink = 1, ) -cc_library( +cc_binary( name = "tensorflow_model_server_main_lib", srcs = [ - "main.cc", ], - hdrs = [ - "version.h", - ], - linkstamp = "version.cc", visibility = [ - ":tensorflow_model_server_custom_op_clients", - "//tensorflow_serving:internal", + "//visibility:public", ], deps = [ ":server_lib", "@org_tensorflow//tensorflow/c:c_api", + "@org_tensorflow//tensorflow/c:c_api_experimental", + "@org_tensorflow//tensorflow/c:kernels_experimental", "@org_tensorflow//tensorflow/compiler/jit:xla_cpu_jit", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", @@ -448,10 +449,12 @@ cc_library( "@org_tensorflow//tensorflow/core/tpu:tpu_api_dlsym_initializer", ]), ), + linkshared = True, ) cc_binary( name = "tensorflow_model_server", + srcs = ["main.cc"], stamp = 1, visibility = [ ":testing", @@ -460,6 +463,7 @@ cc_binary( deps = [ ":tensorflow_model_server_main_lib", ], + linkopts = ["-L/home/gta/jianyi/serving/bazel-bin/tensorflow_serving/model_servers/", "-l:libtensorflow_model_server_main_lib.so"], ) py_test( diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f7be15c2bd8..8ff17ffb5e0 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -45,7 +45,9 @@ limitations under the License. #include #include +#include +#include "tensorflow/c/c_api_experimental.h" #include "tensorflow/c/c_api.h" #include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/lib/core/status.h" @@ -290,15 +292,38 @@ int main(int argc, char** argv) { tensorflow::Flag("thread_pool_factory_config_file", &options.thread_pool_factory_config_file, "If non-empty, read an ascii ThreadPoolConfig protobuf " - "from the supplied file name.")}; + "from the supplied file name."), + tensorflow::Flag("tensorflow_plugins", + &options.tensorflow_plugins, + "Enable tensorflow plugins by giving a path to folder. " + "If non-empty, load all .so files under this folder " + "as tensorflow plugins.")}; const auto& usage = tensorflow::Flags::Usage(argv[0], flag_list); if (!tensorflow::Flags::Parse(&argc, argv, flag_list)) { std::cout << usage; return -1; } - + tensorflow::port::InitMain(argv[0], &argc, &argv); + + if (std::filesystem::exists(options.tensorflow_plugins)){ + for (const auto & entry : std::filesystem::directory_iterator(options.tensorflow_plugins)){ + std::string plugin_file = entry.path().string(); + if (plugin_file.size() > 3 && plugin_file.compare(plugin_file.size() - 3, 3, ".so") == 0) { + TF_Status* plugin_status = TF_NewStatus(); + TF_LoadPluggableDeviceLibrary(entry.path().c_str(), plugin_status); + TF_Code code = TF_GetCode(plugin_status); + if ( code == TF_OK ) { + VLOG(0) << "plugin library "<< entry.path() << " load successfully!"; + } else { + std::string status_msg(TF_Message(plugin_status)); + VLOG(0) << "Could not load " << entry.path() << ": "<< status_msg; + } + } + } + } + #if defined(LIBTPU_ON_GCE) || defined(PLATFORM_CLOUD_TPU) InitializeTPU(options); #endif diff --git a/tensorflow_serving/model_servers/server.h b/tensorflow_serving/model_servers/server.h index 03467d6a1ae..bfbb2ed8899 100644 --- a/tensorflow_serving/model_servers/server.h +++ b/tensorflow_serving/model_servers/server.h @@ -101,6 +101,7 @@ class Server { tensorflow::string thread_pool_factory_config_file; bool enable_signature_method_name_check = false; bool enable_profiler = true; + string tensorflow_plugins = ""; Options(); }; From 1ac2e400d53e768343878b29b731ee68fdabb960 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Thu, 27 Apr 2023 02:02:15 -0700 Subject: [PATCH 02/13] update --- tensorflow_serving/model_servers/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index dc7493b9b7a..d12edcea648 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -463,7 +463,7 @@ cc_binary( deps = [ ":tensorflow_model_server_main_lib", ], - linkopts = ["-L/home/gta/jianyi/serving/bazel-bin/tensorflow_serving/model_servers/", "-l:libtensorflow_model_server_main_lib.so"], + linkopts = ["-Lbazel-out/k8-opt/bin/tensorflow_serving/model_servers/", "-l:libtensorflow_model_server_main_lib.so"], ) py_test( From f3e005c63ff91d3880349d3ca6434424b4b78a94 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Wed, 3 May 2023 20:26:56 -0700 Subject: [PATCH 03/13] c_api.so only, fail --- tensorflow_serving/model_servers/BUILD | 49 +++++++++++++++++--------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index d12edcea648..83a83d79b87 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -39,6 +39,21 @@ filegroup( ), ) +cc_binary( + name = "test", + deps = [ + "@org_tensorflow//tensorflow/c:c_api", + "@org_tensorflow//tensorflow/c:c_api_experimental", + "@org_tensorflow//tensorflow/c:kernels_experimental", + ], + linkshared = 1, +) + +cc_import( + name = "test_lib", + shared_library = ":test", +) + cc_library( name = "model_platform_types", hdrs = ["model_platform_types.h"], @@ -382,12 +397,10 @@ cc_library( srcs = [ "server.cc", ], - hdrs = ["server.h", - "version.h", - ], + hdrs = ["server.h"], visibility = ["//visibility:public"], - linkstamp = "version.cc", deps = [ + ":test_lib", ":grpc_status_util", ":http_server", ":model_platform_types", @@ -414,29 +427,34 @@ cc_library( "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/memory", "@com_google_protobuf//:cc_wkt_protos", - "@org_tensorflow//tensorflow/c:c_api", - "@org_tensorflow//tensorflow/c:c_api_experimental", + #"@org_tensorflow//tensorflow/c:c_api", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core/profiler/rpc:profiler_service_impl", ] + SUPPORTED_TENSORFLOW_OPS, - alwayslink = 1, ) -cc_binary( +cc_library( name = "tensorflow_model_server_main_lib", srcs = [ + "main.cc", ], + hdrs = [ + "version.h", + "@org_tensorflow//tensorflow/c:headers", + "@org_tensorflow//tensorflow/c/eager:headers_2", + ], + linkstamp = "version.cc", visibility = [ - "//visibility:public", + ":tensorflow_model_server_custom_op_clients", + "//tensorflow_serving:internal", ], deps = [ ":server_lib", - "@org_tensorflow//tensorflow/c:c_api", - "@org_tensorflow//tensorflow/c:c_api_experimental", - "@org_tensorflow//tensorflow/c:kernels_experimental", + ":test_lib", + #"@org_tensorflow//tensorflow/c:c_api", "@org_tensorflow//tensorflow/compiler/jit:xla_cpu_jit", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", @@ -449,12 +467,10 @@ cc_binary( "@org_tensorflow//tensorflow/core/tpu:tpu_api_dlsym_initializer", ]), ), - linkshared = True, -) +) cc_binary( name = "tensorflow_model_server", - srcs = ["main.cc"], stamp = 1, visibility = [ ":testing", @@ -463,7 +479,6 @@ cc_binary( deps = [ ":tensorflow_model_server_main_lib", ], - linkopts = ["-Lbazel-out/k8-opt/bin/tensorflow_serving/model_servers/", "-l:libtensorflow_model_server_main_lib.so"], ) py_test( @@ -551,4 +566,4 @@ pkg_deb( maintainer = "TensorFlow Serving team", package = "tensorflow-model-server-universal", version = "undefined", # Set when releasing a new version of TensorFlow Serving (e.g. 1.0.0). -) +) \ No newline at end of file From dce2154f3cd89278b9051478d27a2cbd94ab323c Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Thu, 4 May 2023 05:28:31 -0700 Subject: [PATCH 04/13] can run itex --- tensorflow_serving/model_servers/BUILD | 113 ++++++++++++++++++++----- 1 file changed, 91 insertions(+), 22 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 83a83d79b87..33b771dec38 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -39,21 +39,6 @@ filegroup( ), ) -cc_binary( - name = "test", - deps = [ - "@org_tensorflow//tensorflow/c:c_api", - "@org_tensorflow//tensorflow/c:c_api_experimental", - "@org_tensorflow//tensorflow/c:kernels_experimental", - ], - linkshared = 1, -) - -cc_import( - name = "test_lib", - shared_library = ":test", -) - cc_library( name = "model_platform_types", hdrs = ["model_platform_types.h"], @@ -400,7 +385,6 @@ cc_library( hdrs = ["server.h"], visibility = ["//visibility:public"], deps = [ - ":test_lib", ":grpc_status_util", ":http_server", ":model_platform_types", @@ -427,13 +411,102 @@ cc_library( "@com_github_grpc_grpc//:grpc++", "@com_google_absl//absl/memory", "@com_google_protobuf//:cc_wkt_protos", - #"@org_tensorflow//tensorflow/c:c_api", + "@org_tensorflow//tensorflow/c:c_api", + "@org_tensorflow//tensorflow/c:c_api_experimental", + "@org_tensorflow//tensorflow/c:kernels_experimental", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core/profiler/rpc:profiler_service_impl", ] + SUPPORTED_TENSORFLOW_OPS, + linkopts = [ + "-Wl,--export-dynamic-symbol,TF_AllocateOutput", + "-Wl,--export-dynamic-symbol,TF_AllocateTemp", + "-Wl,--export-dynamic-symbol,TF_AllocateTensor", + "-Wl,--export-dynamic-symbol,TF_DeleteBuffer", + "-Wl,--export-dynamic-symbol,TF_DeleteFunctionLibraryDefinition", + "-Wl,--export-dynamic-symbol,TF_DeleteGraphProperties", + "-Wl,--export-dynamic-symbol,TF_DeleteShapeHandle", + "-Wl,--export-dynamic-symbol,TF_DeleteStatus", + "-Wl,--export-dynamic-symbol,TF_DeleteTensor", + "-Wl,--export-dynamic-symbol,TF_Dim", + "-Wl,--export-dynamic-symbol,TF_ExpectedOutputDataType", + "-Wl,--export-dynamic-symbol,TF_ForwardInputOrAllocateOutput", + "-Wl,--export-dynamic-symbol,TF_GetAllOpList", + "-Wl,--export-dynamic-symbol,TF_GetCode", + "-Wl,--export-dynamic-symbol,TF_GetFetchNodesList", + "-Wl,--export-dynamic-symbol,TF_GetFetchNodesListSize", + "-Wl,--export-dynamic-symbol,TF_GetInput", + "-Wl,--export-dynamic-symbol,TF_GetInputByName", + "-Wl,--export-dynamic-symbol,TF_GetInputPropertiesList", + "-Wl,--export-dynamic-symbol,TF_GetInputPropertiesListSize", + "-Wl,--export-dynamic-symbol,TF_GetInputTensorFromVariable", + "-Wl,--export-dynamic-symbol,TF_GetNodesToPreserveList", + "-Wl,--export-dynamic-symbol,TF_GetNodesToPreserveListSize", + "-Wl,--export-dynamic-symbol,TF_GetOutputPropertiesList", + "-Wl,--export-dynamic-symbol,TF_GetOutputPropertiesListSize", + "-Wl,--export-dynamic-symbol,TF_GetRegisteredKernelsForOp", + "-Wl,--export-dynamic-symbol,TF_InferStatically", + "-Wl,--export-dynamic-symbol,TF_IsRefInput", + "-Wl,--export-dynamic-symbol,TF_KernelBuilder_HostMemory", + "-Wl,--export-dynamic-symbol,TF_KernelBuilder_Priority", + "-Wl,--export-dynamic-symbol,TF_KernelBuilder_TypeConstraint", + "-Wl,--export-dynamic-symbol,TF_LookUpOpDef", + "-Wl,--export-dynamic-symbol,TF_Message", + "-Wl,--export-dynamic-symbol,TF_NewBuffer", + "-Wl,--export-dynamic-symbol,TF_NewFunctionLibraryDefinition", + "-Wl,--export-dynamic-symbol,TF_NewGraphProperties", + "-Wl,--export-dynamic-symbol,TF_NewKernelBuilder", + "-Wl,--export-dynamic-symbol,TF_NewOpDefinitionBuilder", + "-Wl,--export-dynamic-symbol,TF_NewShapeHandle", + "-Wl,--export-dynamic-symbol,TF_NewStatus", + "-Wl,--export-dynamic-symbol,TF_NumDims", + "-Wl,--export-dynamic-symbol,TF_NumInputs", + "-Wl,--export-dynamic-symbol,TF_NumOutputs", + "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddAttr", + "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddInput", + "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddOutput", + "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderSetIsStateful", + "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderSetShapeInferenceFunction", + "-Wl,--export-dynamic-symbol,TF_OpIsStateful", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_Failure", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrBool", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrBoolList", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrFloat", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrFloatList", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt32", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt32List", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt64", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt64List", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrSize", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrString", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrStringList", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrTensorShape", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrType", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrTypeList", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetName", + "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_HasAttr", + "-Wl,--export-dynamic-symbol,TF_OpKernelContext_Failure", + "-Wl,--export-dynamic-symbol,TF_OpKernelContext_ForwardRefInputToRefOutput", + "-Wl,--export-dynamic-symbol,TF_RegisterKernelBuilder", + "-Wl,--export-dynamic-symbol,TF_RegisterOpDefinition", + "-Wl,--export-dynamic-symbol,TF_SetOutput", + "-Wl,--export-dynamic-symbol,TF_SetStatus", + "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextConcatenateShapes", + "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextGetInput", + "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSetOutput", + "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSetUnknownShape", + "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSubshape", + "-Wl,--export-dynamic-symbol,TF_StepId", + "-Wl,--export-dynamic-symbol,TF_TensorBitcastFrom", + "-Wl,--export-dynamic-symbol,TF_TensorData", + "-Wl,--export-dynamic-symbol,TF_TensorFromProto", + "-Wl,--export-dynamic-symbol,TF_TensorIsAligned", + "-Wl,--export-dynamic-symbol,TF_TensorMaybeMove", + "-Wl,--export-dynamic-symbol,TF_TensorType", + "-Wl,--export-dynamic-symbol,TF_Version", + ], ) cc_library( @@ -443,8 +516,6 @@ cc_library( ], hdrs = [ "version.h", - "@org_tensorflow//tensorflow/c:headers", - "@org_tensorflow//tensorflow/c/eager:headers_2", ], linkstamp = "version.cc", visibility = [ @@ -453,8 +524,6 @@ cc_library( ], deps = [ ":server_lib", - ":test_lib", - #"@org_tensorflow//tensorflow/c:c_api", "@org_tensorflow//tensorflow/compiler/jit:xla_cpu_jit", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", @@ -467,7 +536,7 @@ cc_library( "@org_tensorflow//tensorflow/core/tpu:tpu_api_dlsym_initializer", ]), ), -) +) cc_binary( name = "tensorflow_model_server", From 8587bef7ccc8567535d6ef6e79980b33c38ac26c Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Thu, 4 May 2023 22:36:43 -0700 Subject: [PATCH 05/13] use linkopt --dynamic-list to export all matched symbols --- tensorflow_serving/model_servers/BUILD | 93 +------------------ .../model_servers/tf_exported_symbols.lds | 17 ++++ 2 files changed, 22 insertions(+), 88 deletions(-) create mode 100644 tensorflow_serving/model_servers/tf_exported_symbols.lds diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 33b771dec38..e5ae8f5329e 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -383,6 +383,9 @@ cc_library( "server.cc", ], hdrs = ["server.h"], + linkopts = [ + "-Wl,--dynamic-list,$(location //tensorflow_serving/model_servers:tf_exported_symbols.lds)", + ], visibility = ["//visibility:public"], deps = [ ":grpc_status_util", @@ -397,6 +400,7 @@ cc_library( "//tensorflow_serving/config:platform_config_cc_proto", "//tensorflow_serving/config:ssl_config_cc_proto", "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/model_servers:tf_exported_symbols.lds", "//tensorflow_serving/servables/tensorflow:classification_service", "//tensorflow_serving/servables/tensorflow:get_model_metadata_impl", "//tensorflow_serving/servables/tensorflow:multi_inference", @@ -420,93 +424,6 @@ cc_library( "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core/profiler/rpc:profiler_service_impl", ] + SUPPORTED_TENSORFLOW_OPS, - linkopts = [ - "-Wl,--export-dynamic-symbol,TF_AllocateOutput", - "-Wl,--export-dynamic-symbol,TF_AllocateTemp", - "-Wl,--export-dynamic-symbol,TF_AllocateTensor", - "-Wl,--export-dynamic-symbol,TF_DeleteBuffer", - "-Wl,--export-dynamic-symbol,TF_DeleteFunctionLibraryDefinition", - "-Wl,--export-dynamic-symbol,TF_DeleteGraphProperties", - "-Wl,--export-dynamic-symbol,TF_DeleteShapeHandle", - "-Wl,--export-dynamic-symbol,TF_DeleteStatus", - "-Wl,--export-dynamic-symbol,TF_DeleteTensor", - "-Wl,--export-dynamic-symbol,TF_Dim", - "-Wl,--export-dynamic-symbol,TF_ExpectedOutputDataType", - "-Wl,--export-dynamic-symbol,TF_ForwardInputOrAllocateOutput", - "-Wl,--export-dynamic-symbol,TF_GetAllOpList", - "-Wl,--export-dynamic-symbol,TF_GetCode", - "-Wl,--export-dynamic-symbol,TF_GetFetchNodesList", - "-Wl,--export-dynamic-symbol,TF_GetFetchNodesListSize", - "-Wl,--export-dynamic-symbol,TF_GetInput", - "-Wl,--export-dynamic-symbol,TF_GetInputByName", - "-Wl,--export-dynamic-symbol,TF_GetInputPropertiesList", - "-Wl,--export-dynamic-symbol,TF_GetInputPropertiesListSize", - "-Wl,--export-dynamic-symbol,TF_GetInputTensorFromVariable", - "-Wl,--export-dynamic-symbol,TF_GetNodesToPreserveList", - "-Wl,--export-dynamic-symbol,TF_GetNodesToPreserveListSize", - "-Wl,--export-dynamic-symbol,TF_GetOutputPropertiesList", - "-Wl,--export-dynamic-symbol,TF_GetOutputPropertiesListSize", - "-Wl,--export-dynamic-symbol,TF_GetRegisteredKernelsForOp", - "-Wl,--export-dynamic-symbol,TF_InferStatically", - "-Wl,--export-dynamic-symbol,TF_IsRefInput", - "-Wl,--export-dynamic-symbol,TF_KernelBuilder_HostMemory", - "-Wl,--export-dynamic-symbol,TF_KernelBuilder_Priority", - "-Wl,--export-dynamic-symbol,TF_KernelBuilder_TypeConstraint", - "-Wl,--export-dynamic-symbol,TF_LookUpOpDef", - "-Wl,--export-dynamic-symbol,TF_Message", - "-Wl,--export-dynamic-symbol,TF_NewBuffer", - "-Wl,--export-dynamic-symbol,TF_NewFunctionLibraryDefinition", - "-Wl,--export-dynamic-symbol,TF_NewGraphProperties", - "-Wl,--export-dynamic-symbol,TF_NewKernelBuilder", - "-Wl,--export-dynamic-symbol,TF_NewOpDefinitionBuilder", - "-Wl,--export-dynamic-symbol,TF_NewShapeHandle", - "-Wl,--export-dynamic-symbol,TF_NewStatus", - "-Wl,--export-dynamic-symbol,TF_NumDims", - "-Wl,--export-dynamic-symbol,TF_NumInputs", - "-Wl,--export-dynamic-symbol,TF_NumOutputs", - "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddAttr", - "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddInput", - "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderAddOutput", - "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderSetIsStateful", - "-Wl,--export-dynamic-symbol,TF_OpDefinitionBuilderSetShapeInferenceFunction", - "-Wl,--export-dynamic-symbol,TF_OpIsStateful", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_Failure", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrBool", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrBoolList", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrFloat", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrFloatList", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt32", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt32List", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt64", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrInt64List", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrSize", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrString", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrStringList", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrTensorShape", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrType", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetAttrTypeList", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_GetName", - "-Wl,--export-dynamic-symbol,TF_OpKernelConstruction_HasAttr", - "-Wl,--export-dynamic-symbol,TF_OpKernelContext_Failure", - "-Wl,--export-dynamic-symbol,TF_OpKernelContext_ForwardRefInputToRefOutput", - "-Wl,--export-dynamic-symbol,TF_RegisterKernelBuilder", - "-Wl,--export-dynamic-symbol,TF_RegisterOpDefinition", - "-Wl,--export-dynamic-symbol,TF_SetOutput", - "-Wl,--export-dynamic-symbol,TF_SetStatus", - "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextConcatenateShapes", - "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextGetInput", - "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSetOutput", - "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSetUnknownShape", - "-Wl,--export-dynamic-symbol,TF_ShapeInferenceContextSubshape", - "-Wl,--export-dynamic-symbol,TF_StepId", - "-Wl,--export-dynamic-symbol,TF_TensorBitcastFrom", - "-Wl,--export-dynamic-symbol,TF_TensorData", - "-Wl,--export-dynamic-symbol,TF_TensorFromProto", - "-Wl,--export-dynamic-symbol,TF_TensorIsAligned", - "-Wl,--export-dynamic-symbol,TF_TensorMaybeMove", - "-Wl,--export-dynamic-symbol,TF_TensorType", - "-Wl,--export-dynamic-symbol,TF_Version", - ], ) cc_library( @@ -635,4 +552,4 @@ pkg_deb( maintainer = "TensorFlow Serving team", package = "tensorflow-model-server-universal", version = "undefined", # Set when releasing a new version of TensorFlow Serving (e.g. 1.0.0). -) \ No newline at end of file +) diff --git a/tensorflow_serving/model_servers/tf_exported_symbols.lds b/tensorflow_serving/model_servers/tf_exported_symbols.lds new file mode 100644 index 00000000000..dedb4a06fc4 --- /dev/null +++ b/tensorflow_serving/model_servers/tf_exported_symbols.lds @@ -0,0 +1,17 @@ +{ + *tensorflow*; + *absl*kSeed*; + *toco*; + *perftools*gputools*; + *tf_*; + *TF_*; + *Eager*; + *TFE_*; + *nsync_*; + *stream_executor*; + *xla*; + *PyInit_*; + *SE_*; + *SP_*; + *TFL*; +}; \ No newline at end of file From b50ab18c2b736ea0e2c5a3ed61be20118323371e Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Fri, 5 May 2023 05:46:26 -0700 Subject: [PATCH 06/13] Auto generate symbol based on TF. --- WORKSPACE | 7 +-- tensorflow_serving/model_servers/BUILD | 25 +++++++++-- .../model_servers/gen_tf_symbols.py | 43 +++++++++++++++++++ .../model_servers/tf_exported_symbols.lds | 17 -------- tensorflow_serving/serving.bzl | 13 ++++++ 5 files changed, 81 insertions(+), 24 deletions(-) create mode 100644 tensorflow_serving/model_servers/gen_tf_symbols.py delete mode 100644 tensorflow_serving/model_servers/tf_exported_symbols.lds diff --git a/WORKSPACE b/WORKSPACE index 1eb96757e55..8c9ff35dab0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -21,10 +21,11 @@ local_repository( # and update the 'sha256' arg with the result. # 3. Request the new archive to be mirrored on mirror.bazel.build for more # reliable downloads. -#load("//tensorflow_serving:repo.bzl") -local_repository( +load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") +tensorflow_http_archive( name = "org_tensorflow", - path = "/home/gta/jianyi/tensorflow", + sha256 = "4d4f15fa4e52f7c77a1793656f85d260225e2638aee34a9c7625583e0608265c", + git_commit = "332077006a5ed7004ee19fbae306768f8f500ca8", ) # Import all of TensorFlow Serving's external dependencies. diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index e5ae8f5329e..7ffe7a5f014 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -1,5 +1,6 @@ # Description: Model Server +load("//tensorflow_serving:serving.bzl", "export_tf_symbol") load("//tensorflow_serving:tensorflow_version.bzl", "if_not_v2", "if_v2") load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar") load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_google", "if_libtpu", "if_with_tpu_support") @@ -383,9 +384,7 @@ cc_library( "server.cc", ], hdrs = ["server.h"], - linkopts = [ - "-Wl,--dynamic-list,$(location //tensorflow_serving/model_servers:tf_exported_symbols.lds)", - ], + textual_hdrs = [":tf_exported_symbols"], visibility = ["//visibility:public"], deps = [ ":grpc_status_util", @@ -400,7 +399,6 @@ cc_library( "//tensorflow_serving/config:platform_config_cc_proto", "//tensorflow_serving/config:ssl_config_cc_proto", "//tensorflow_serving/core:availability_preserving_policy", - "//tensorflow_serving/model_servers:tf_exported_symbols.lds", "//tensorflow_serving/servables/tensorflow:classification_service", "//tensorflow_serving/servables/tensorflow:get_model_metadata_impl", "//tensorflow_serving/servables/tensorflow:multi_inference", @@ -457,6 +455,12 @@ cc_library( cc_binary( name = "tensorflow_model_server", + additional_linker_inputs = [ + ":tf_exported_symbols.lds", + ], + linkopts = [ + "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)" + ], stamp = 1, visibility = [ ":testing", @@ -524,6 +528,19 @@ py_binary( srcs_version = "PY2AND3", ) +py_binary( + name = "gen_tf_symbols", + srcs = ["gen_tf_symbols.py"], + visibility = ["//visibility:public"], +) + +# Create symbol file based on TensorFlow. +export_tf_symbol( + name = "tf_exported_symbols", + header_in = "@org_tensorflow//tensorflow:tf_exported_symbols.lds", + header_out = ":tf_exported_symbols.lds", +) + pkg_tar( name = "tensorflow_model_server_tar", srcs = [ diff --git a/tensorflow_serving/model_servers/gen_tf_symbols.py b/tensorflow_serving/model_servers/gen_tf_symbols.py new file mode 100644 index 00000000000..f6d5ba6af1d --- /dev/null +++ b/tensorflow_serving/model_servers/gen_tf_symbols.py @@ -0,0 +1,43 @@ +import os +import sys + + +def parse_args(argv): + result = {} + for arg in argv: + k, v = arg.split("=") + result[k] = v + + return result + + +def generate_version(header_in, header_out): + header_out = os.path.expanduser(header_out) + header_out_dir = os.path.dirname(header_out) + + if not os.path.exists(header_out_dir): + os.makedirs(header_out_dir, exist_ok=True) + + with open(header_out, "w") as outf: + outf.write("{\n") + with open(os.path.expanduser(header_in)) as inf: + content = inf.readlines() + + for line in content: + line = line.strip().strip(";") + + # Excluding below symbols to avoid link warining. + if line.find("lite") != -1 or line.find("tsl") != -1: + continue + outf.writelines(" " + line + ";\n") + + outf.write("};") + + +def main(): + args = parse_args(sys.argv[1:]) + generate_version(args["--in"], args["--out"]) + + +if __name__ == "__main__": + main() diff --git a/tensorflow_serving/model_servers/tf_exported_symbols.lds b/tensorflow_serving/model_servers/tf_exported_symbols.lds deleted file mode 100644 index dedb4a06fc4..00000000000 --- a/tensorflow_serving/model_servers/tf_exported_symbols.lds +++ /dev/null @@ -1,17 +0,0 @@ -{ - *tensorflow*; - *absl*kSeed*; - *toco*; - *perftools*gputools*; - *tf_*; - *TF_*; - *Eager*; - *TFE_*; - *nsync_*; - *stream_executor*; - *xla*; - *PyInit_*; - *SE_*; - *SP_*; - *TFL*; -}; \ No newline at end of file diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index 76eb704a9e1..d89691b6b27 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -60,3 +60,16 @@ def serving_tensorflow_proto_dep(dep): """Rename for deps onto tensorflow protos in serving_proto_library targets. """ return "{}_cc".format(dep) + +def export_tf_symbol(name, header_in, header_out, **kwargs): + tool = "//tensorflow_serving/model_servers:gen_tf_symbols" + + native.genrule( + name = name, + srcs = [header_in], + outs = [header_out], + tools = [tool], + cmd = "$(location {}) ".format(tool) + "--in=$< " + "--out=$@", + stamp = True, + **kwargs + ) From 5b9b5a6eb869dcbd129deebb48e1af8e898f9acd Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Sun, 7 May 2023 19:11:52 -0700 Subject: [PATCH 07/13] code style --- tensorflow_serving/model_servers/BUILD | 2 +- tensorflow_serving/model_servers/main.cc | 27 ++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 7ffe7a5f014..b802e6cc876 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -459,7 +459,7 @@ cc_binary( ":tf_exported_symbols.lds", ], linkopts = [ - "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)" + "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)", ], stamp = 1, visibility = [ diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 8ff17ffb5e0..bf53e8d8559 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -43,12 +43,12 @@ limitations under the License. // To enable batching (default disabled): --enable_batching // To override the default batching parameters: --batching_parameters_file +#include #include #include -#include -#include "tensorflow/c/c_api_experimental.h" #include "tensorflow/c/c_api.h" +#include "tensorflow/c/c_api_experimental.h" #include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/init_main.h" @@ -76,7 +76,7 @@ void InitializeTPU(tensorflow::serving::main::Server::Options& server_options) { } #endif -int main(int argc, char** argv) { +int main(int argc, char **argv) { tensorflow::serving::main::Server::Options options; bool display_version = false; bool xla_cpu_compilation_enabled = false; @@ -293,8 +293,7 @@ int main(int argc, char** argv) { &options.thread_pool_factory_config_file, "If non-empty, read an ascii ThreadPoolConfig protobuf " "from the supplied file name."), - tensorflow::Flag("tensorflow_plugins", - &options.tensorflow_plugins, + tensorflow::Flag("tensorflow_plugins", &options.tensorflow_plugins, "Enable tensorflow plugins by giving a path to folder. " "If non-empty, load all .so files under this folder " "as tensorflow plugins.")}; @@ -304,21 +303,23 @@ int main(int argc, char** argv) { std::cout << usage; return -1; } - + tensorflow::port::InitMain(argv[0], &argc, &argv); - if (std::filesystem::exists(options.tensorflow_plugins)){ - for (const auto & entry : std::filesystem::directory_iterator(options.tensorflow_plugins)){ + if (std::filesystem::exists(options.tensorflow_plugins)) { + for (const auto &entry : + std::filesystem::directory_iterator(options.tensorflow_plugins)) { std::string plugin_file = entry.path().string(); - if (plugin_file.size() > 3 && plugin_file.compare(plugin_file.size() - 3, 3, ".so") == 0) { - TF_Status* plugin_status = TF_NewStatus(); + if (plugin_file.size() > 3 && + plugin_file.compare(plugin_file.size() - 3, 3, ".so") == 0) { + TF_Status *plugin_status = TF_NewStatus(); TF_LoadPluggableDeviceLibrary(entry.path().c_str(), plugin_status); TF_Code code = TF_GetCode(plugin_status); - if ( code == TF_OK ) { - VLOG(0) << "plugin library "<< entry.path() << " load successfully!"; + if (code == TF_OK) { + VLOG(0) << "plugin library " << entry.path() << " load successfully!"; } else { std::string status_msg(TF_Message(plugin_status)); - VLOG(0) << "Could not load " << entry.path() << ": "<< status_msg; + VLOG(0) << "Could not load " << entry.path() << ": " << status_msg; } } } From 49a02e0ddc46042c19afa8a4710fe7cb2acecffe Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Sun, 7 May 2023 22:20:31 -0700 Subject: [PATCH 08/13] add build option for tf plugin --- tensorflow_serving/BUILD | 28 +++++++++++++++++++++++ tensorflow_serving/model_servers/BUILD | 25 ++++++++++++-------- tensorflow_serving/model_servers/main.cc | 16 +++++++++---- tensorflow_serving/model_servers/server.h | 2 ++ tensorflow_serving/serving.bzl | 7 ++++++ 5 files changed, 64 insertions(+), 14 deletions(-) diff --git a/tensorflow_serving/BUILD b/tensorflow_serving/BUILD index 7a017679afb..dec2dad33ed 100644 --- a/tensorflow_serving/BUILD +++ b/tensorflow_serving/BUILD @@ -1,4 +1,6 @@ # Description: Tensorflow Serving. +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_visibility = ["//tensorflow_serving:internal"]) @@ -24,3 +26,29 @@ filegroup( ], ), ) + +bool_flag( + name = "enable_plugins_support", + build_setting_default = False, +) + +config_setting( + name = "with_plugins_support_define", + define_values = {"with_plugins_support": "true"}, + visibility = ["//visibility:public"], +) + +config_setting( + name = "with_plugins_support_flag", + flag_values = {":enable_plugins_support": "True"}, + visibility = ["//visibility:public"], +) + +selects.config_setting_group( + name = "with_plugins_support", + match_any = [ + ":with_plugins_support_define", + ":with_plugins_support_flag", + ], + visibility = ["//visibility:public"], +) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index b802e6cc876..a3eb2e9e0ca 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -1,6 +1,6 @@ # Description: Model Server -load("//tensorflow_serving:serving.bzl", "export_tf_symbol") +load("//tensorflow_serving:serving.bzl", "export_tf_symbol", "if_with_plugins_support") load("//tensorflow_serving:tensorflow_version.bzl", "if_not_v2", "if_v2") load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar") load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_google", "if_libtpu", "if_with_tpu_support") @@ -414,14 +414,17 @@ cc_library( "@com_google_absl//absl/memory", "@com_google_protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/c:c_api", - "@org_tensorflow//tensorflow/c:c_api_experimental", - "@org_tensorflow//tensorflow/c:kernels_experimental", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core/profiler/rpc:profiler_service_impl", - ] + SUPPORTED_TENSORFLOW_OPS, + ] + SUPPORTED_TENSORFLOW_OPS + if_with_plugins_support([ + "@org_tensorflow//tensorflow/c:c_api_experimental", + "@org_tensorflow//tensorflow/c:kernels_experimental", + "@org_tensorflow//tensorflow/c:ops", + "@org_tensorflow//tensorflow/c/experimental/next_pluggable_device:c_api", + ]), ) cc_library( @@ -455,12 +458,14 @@ cc_library( cc_binary( name = "tensorflow_model_server", - additional_linker_inputs = [ - ":tf_exported_symbols.lds", - ], - linkopts = [ - "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)", - ], + additional_linker_inputs = + if_with_plugins_support([ + ":tf_exported_symbols.lds", + ]), + linkopts = + if_with_plugins_support([ + "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)", + ]), stamp = 1, visibility = [ ":testing", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index bf53e8d8559..a46775be3d8 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -43,12 +43,10 @@ limitations under the License. // To enable batching (default disabled): --enable_batching // To override the default batching parameters: --batching_parameters_file -#include #include #include #include "tensorflow/c/c_api.h" -#include "tensorflow/c/c_api_experimental.h" #include "tensorflow/compiler/jit/flags.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/init_main.h" @@ -76,6 +74,11 @@ void InitializeTPU(tensorflow::serving::main::Server::Options& server_options) { } #endif +#ifdef SUPPORT_TF_PLUGINS +#include +#include "tensorflow/c/c_api_experimental.h" +#endif + int main(int argc, char **argv) { tensorflow::serving::main::Server::Options options; bool display_version = false; @@ -293,12 +296,15 @@ int main(int argc, char **argv) { &options.thread_pool_factory_config_file, "If non-empty, read an ascii ThreadPoolConfig protobuf " "from the supplied file name."), +#ifdef SUPPORT_TF_PLUGINS tensorflow::Flag("tensorflow_plugins", &options.tensorflow_plugins, "Enable tensorflow plugins by giving a path to folder. " "If non-empty, load all .so files under this folder " - "as tensorflow plugins.")}; + "as tensorflow plugins.") +#endif + }; - const auto& usage = tensorflow::Flags::Usage(argv[0], flag_list); + const auto &usage = tensorflow::Flags::Usage(argv[0], flag_list); if (!tensorflow::Flags::Parse(&argc, argv, flag_list)) { std::cout << usage; return -1; @@ -306,6 +312,7 @@ int main(int argc, char **argv) { tensorflow::port::InitMain(argv[0], &argc, &argv); +#ifdef SUPPORT_TF_PLUGINS if (std::filesystem::exists(options.tensorflow_plugins)) { for (const auto &entry : std::filesystem::directory_iterator(options.tensorflow_plugins)) { @@ -324,6 +331,7 @@ int main(int argc, char **argv) { } } } +#endif #if defined(LIBTPU_ON_GCE) || defined(PLATFORM_CLOUD_TPU) InitializeTPU(options); diff --git a/tensorflow_serving/model_servers/server.h b/tensorflow_serving/model_servers/server.h index bfbb2ed8899..26dfdb999c7 100644 --- a/tensorflow_serving/model_servers/server.h +++ b/tensorflow_serving/model_servers/server.h @@ -101,7 +101,9 @@ class Server { tensorflow::string thread_pool_factory_config_file; bool enable_signature_method_name_check = false; bool enable_profiler = true; +#ifdef SUPPORT_TF_PLUGINS string tensorflow_plugins = ""; +#endif Options(); }; diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index d89691b6b27..c4217ee801b 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -73,3 +73,10 @@ def export_tf_symbol(name, header_in, header_out, **kwargs): stamp = True, **kwargs ) + +def if_with_plugins_support(if_true, if_false = []): + """Shorthand for select()ing whether to build API support for TensorFlow Plugins""" + return select({ + "//tensorflow_serving:with_plugins_support": if_true, + "//conditions:default": if_false, + }) From 2cae7f0d1229bd5e1286a7a4266ca1c71fb52149 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Sun, 7 May 2023 22:47:05 -0700 Subject: [PATCH 09/13] remove unnecessary config setting --- .bazelrc | 2 ++ tensorflow_serving/BUILD | 23 +---------------------- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/.bazelrc b/.bazelrc index 085f520ba59..ab2f280aa03 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,6 +41,8 @@ build:nativeopt --copt=-march=native build:nativeopt --host_copt=-march=native build:nativeopt --copt=-O3 +build:plugins --copt=-DSUPPORT_TF_PLUGINS --define=with_plugins_support=true + build --keep_going build --verbose_failures=true build --spawn_strategy=standalone diff --git a/tensorflow_serving/BUILD b/tensorflow_serving/BUILD index dec2dad33ed..270d594ee18 100644 --- a/tensorflow_serving/BUILD +++ b/tensorflow_serving/BUILD @@ -1,6 +1,4 @@ # Description: Tensorflow Serving. -load("@bazel_skylib//lib:selects.bzl", "selects") -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_visibility = ["//tensorflow_serving:internal"]) @@ -27,28 +25,9 @@ filegroup( ), ) -bool_flag( - name = "enable_plugins_support", - build_setting_default = False, -) - -config_setting( - name = "with_plugins_support_define", - define_values = {"with_plugins_support": "true"}, - visibility = ["//visibility:public"], -) config_setting( - name = "with_plugins_support_flag", - flag_values = {":enable_plugins_support": "True"}, - visibility = ["//visibility:public"], -) - -selects.config_setting_group( name = "with_plugins_support", - match_any = [ - ":with_plugins_support_define", - ":with_plugins_support_flag", - ], + define_values = {"with_plugins_support": "true"}, visibility = ["//visibility:public"], ) From ec47bb932a406d6ee6bfad81fc6b17385f175ae3 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Mon, 8 May 2023 01:06:31 -0700 Subject: [PATCH 10/13] make pluggin device support default --- .bazelrc | 3 ++- tensorflow_serving/model_servers/BUILD | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bazelrc b/.bazelrc index ab2f280aa03..b2ff4447463 100644 --- a/.bazelrc +++ b/.bazelrc @@ -41,7 +41,8 @@ build:nativeopt --copt=-march=native build:nativeopt --host_copt=-march=native build:nativeopt --copt=-O3 -build:plugins --copt=-DSUPPORT_TF_PLUGINS --define=with_plugins_support=true +# Support TF pluggable device +build --copt=-DSUPPORT_TF_PLUGINS --define=with_plugins_support=true build --keep_going build --verbose_failures=true diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index a3eb2e9e0ca..505c687fff1 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -422,7 +422,6 @@ cc_library( ] + SUPPORTED_TENSORFLOW_OPS + if_with_plugins_support([ "@org_tensorflow//tensorflow/c:c_api_experimental", "@org_tensorflow//tensorflow/c:kernels_experimental", - "@org_tensorflow//tensorflow/c:ops", "@org_tensorflow//tensorflow/c/experimental/next_pluggable_device:c_api", ]), ) From 0988ec6313d5cbbec2c203d9f5c40820e4a577d4 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Mon, 8 May 2023 01:38:49 -0700 Subject: [PATCH 11/13] export only *TF_* symbols --- tensorflow_serving/model_servers/BUILD | 21 ++------- .../model_servers/gen_tf_symbols.py | 43 ------------------- tensorflow_serving/model_servers/main.cc | 2 +- .../tf_c_api_exported_symbols.lds | 3 ++ tensorflow_serving/serving.bzl | 13 ------ 5 files changed, 8 insertions(+), 74 deletions(-) delete mode 100644 tensorflow_serving/model_servers/gen_tf_symbols.py create mode 100644 tensorflow_serving/model_servers/tf_c_api_exported_symbols.lds diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 505c687fff1..5dda9039721 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -1,6 +1,6 @@ # Description: Model Server -load("//tensorflow_serving:serving.bzl", "export_tf_symbol", "if_with_plugins_support") +load("//tensorflow_serving:serving.bzl", "if_with_plugins_support") load("//tensorflow_serving:tensorflow_version.bzl", "if_not_v2", "if_v2") load("@rules_pkg//:pkg.bzl", "pkg_deb", "pkg_tar") load("@org_tensorflow//tensorflow:tensorflow.bzl", "if_google", "if_libtpu", "if_with_tpu_support") @@ -384,7 +384,7 @@ cc_library( "server.cc", ], hdrs = ["server.h"], - textual_hdrs = [":tf_exported_symbols"], + textual_hdrs = ["tf_c_api_exported_symbols.lds"], visibility = ["//visibility:public"], deps = [ ":grpc_status_util", @@ -459,11 +459,11 @@ cc_binary( name = "tensorflow_model_server", additional_linker_inputs = if_with_plugins_support([ - ":tf_exported_symbols.lds", + "tf_c_api_exported_symbols.lds", ]), linkopts = if_with_plugins_support([ - "-Wl,-dynamic-list,$(location :tf_exported_symbols.lds)", + "-Wl,-dynamic-list,$(location :tf_c_api_exported_symbols.lds)", ]), stamp = 1, visibility = [ @@ -532,19 +532,6 @@ py_binary( srcs_version = "PY2AND3", ) -py_binary( - name = "gen_tf_symbols", - srcs = ["gen_tf_symbols.py"], - visibility = ["//visibility:public"], -) - -# Create symbol file based on TensorFlow. -export_tf_symbol( - name = "tf_exported_symbols", - header_in = "@org_tensorflow//tensorflow:tf_exported_symbols.lds", - header_out = ":tf_exported_symbols.lds", -) - pkg_tar( name = "tensorflow_model_server_tar", srcs = [ diff --git a/tensorflow_serving/model_servers/gen_tf_symbols.py b/tensorflow_serving/model_servers/gen_tf_symbols.py deleted file mode 100644 index f6d5ba6af1d..00000000000 --- a/tensorflow_serving/model_servers/gen_tf_symbols.py +++ /dev/null @@ -1,43 +0,0 @@ -import os -import sys - - -def parse_args(argv): - result = {} - for arg in argv: - k, v = arg.split("=") - result[k] = v - - return result - - -def generate_version(header_in, header_out): - header_out = os.path.expanduser(header_out) - header_out_dir = os.path.dirname(header_out) - - if not os.path.exists(header_out_dir): - os.makedirs(header_out_dir, exist_ok=True) - - with open(header_out, "w") as outf: - outf.write("{\n") - with open(os.path.expanduser(header_in)) as inf: - content = inf.readlines() - - for line in content: - line = line.strip().strip(";") - - # Excluding below symbols to avoid link warining. - if line.find("lite") != -1 or line.find("tsl") != -1: - continue - outf.writelines(" " + line + ";\n") - - outf.write("};") - - -def main(): - args = parse_args(sys.argv[1:]) - generate_version(args["--in"], args["--out"]) - - -if __name__ == "__main__": - main() diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index a46775be3d8..81c0df218ec 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -304,7 +304,7 @@ int main(int argc, char **argv) { #endif }; - const auto &usage = tensorflow::Flags::Usage(argv[0], flag_list); + const auto& usage = tensorflow::Flags::Usage(argv[0], flag_list); if (!tensorflow::Flags::Parse(&argc, argv, flag_list)) { std::cout << usage; return -1; diff --git a/tensorflow_serving/model_servers/tf_c_api_exported_symbols.lds b/tensorflow_serving/model_servers/tf_c_api_exported_symbols.lds new file mode 100644 index 00000000000..b5e82a097c2 --- /dev/null +++ b/tensorflow_serving/model_servers/tf_c_api_exported_symbols.lds @@ -0,0 +1,3 @@ +{ + *TF_*; +}; diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index c4217ee801b..04bec784eb1 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -61,19 +61,6 @@ def serving_tensorflow_proto_dep(dep): """ return "{}_cc".format(dep) -def export_tf_symbol(name, header_in, header_out, **kwargs): - tool = "//tensorflow_serving/model_servers:gen_tf_symbols" - - native.genrule( - name = name, - srcs = [header_in], - outs = [header_out], - tools = [tool], - cmd = "$(location {}) ".format(tool) + "--in=$< " + "--out=$@", - stamp = True, - **kwargs - ) - def if_with_plugins_support(if_true, if_false = []): """Shorthand for select()ing whether to build API support for TensorFlow Plugins""" return select({ From 5259247581076f7a3357835e7183aef3407dc7a2 Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Mon, 15 May 2023 20:55:28 -0700 Subject: [PATCH 12/13] remove unnecessay textual_hdrs --- tensorflow_serving/model_servers/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 5dda9039721..3045c42ff50 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -384,7 +384,6 @@ cc_library( "server.cc", ], hdrs = ["server.h"], - textual_hdrs = ["tf_c_api_exported_symbols.lds"], visibility = ["//visibility:public"], deps = [ ":grpc_status_util", From c179a5b5a794eb7fd7d549179fddd5d5db24078e Mon Sep 17 00:00:00 2001 From: "Zhang, Jianyi" Date: Thu, 12 Oct 2023 23:06:40 -0700 Subject: [PATCH 13/13] fix --- tensorflow_serving/model_servers/BUILD | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 8f4e77ca25f..48ccb571b2a 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -466,10 +466,7 @@ cc_binary( if_with_plugins_support([ "tf_c_api_exported_symbols.lds", ]), - linkopts = [ - # Exports Tensorflow APIs - "-rdynamic", - ] + if_with_plugins_support([ + linkopts = if_with_plugins_support([ "-Wl,-dynamic-list,$(location :tf_c_api_exported_symbols.lds)", ]), stamp = 1,