Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add ernie-3.0 mkldnn fp32 and int8 support #2468

Merged
merged 4 commits into from
Jun 13, 2022
Merged

Add ernie-3.0 mkldnn fp32 and int8 support #2468

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
59d6067
add ernie-3.0 mkldnn fp32 and int8 support
lidanqing-intel Jun 4, 2022
b977edb
fix code style
lidanqing-intel Jun 9, 2022
b034a7a
fix doc
lidanqing-intel Jun 10, 2022
65a4fd4
update
lidanqing-intel Jun 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions model_zoo/ernie-3.0/infer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,12 +121,24 @@ def parse_args():
parser.add_argument(
"--enable_quantize",
action='store_true',
help="Whether to enable quantization for acceleration.", )
help="Whether to enable quantization for acceleration. Valid for both onnx and dnnl",
)
parser.add_argument(
"--use_onnxruntime",
type=distutils.util.strtobool,
default=False,
help="Use onnxruntime to infer or not.")
parser.add_argument(
"--debug",
action='store_true',
help="With debug it will save graph and model after each pass.")
parser.add_argument(
"--provider",
default='CPUExecutionProvider',
choices=['CPUExecutionProvider', 'DnnlExecutionProvider'],
type=str,
help="Onnx ExecutionProvider with DNNL or without DNNL")

args = parser.parse_args()
return args

Expand Down Expand Up @@ -204,23 +216,22 @@ def create_predictor(cls, args):
opset_version=13,
enable_onnx_checker=True)
dynamic_quantize_model = onnx_model
providers = ['CUDAExecutionProvider']
if args.enable_quantize:
from onnxruntime.quantization import QuantizationMode, quantize_dynamic

float_onnx_file = "model.onnx"
with open(float_onnx_file, "wb") as f:
f.write(onnx_model)
dynamic_quantize_model = "dynamic_quantize_model.onnx"
quantize_dynamic(float_onnx_file, dynamic_quantize_model)
providers = ['CPUExecutionProvider']
sess_options = ort.SessionOptions()
sess_options.intra_op_num_threads = args.num_threads
sess_options.inter_op_num_threads = args.num_threads
executionprovider = args.provider
print("ExecutionProvider is: ", executionprovider)
predictor = ort.InferenceSession(
dynamic_quantize_model,
sess_options=sess_options,
providers=providers)
providers=[executionprovider])
input_name1 = predictor.get_inputs()[1].name
input_name2 = predictor.get_inputs()[0].name
input_handles = [input_name1, input_name2]
Expand All @@ -238,6 +249,10 @@ def create_predictor(cls, args):
config.disable_gpu()
config.switch_ir_optim(True)
config.enable_mkldnn()
if args.enable_quantize:
config.enable_mkldnn_int8()
if args.debug:
config.switch_ir_debug(True)
config.set_cpu_math_library_num_threads(args.num_threads)
cls.device = paddle.set_device("cpu")
elif args.device == "xpu":
Expand Down