From b16c9fc339d779ad4bcdc70c17cde8185f2edabb Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 15 Dec 2021 22:48:47 +0800 Subject: [PATCH] [Frontend][PaddlePaddle] Enhance paddlepaddle frontend with more operators (#9724) * add operators for paddle frontend * add operators for paddle frontend * retrigger ci * retrigger ci * retrigger ci Co-authored-by: wjj19950828 --- python/tvm/relay/frontend/paddlepaddle.py | 351 ++++++++++++++++-- .../frontend/paddlepaddle/test_forward.py | 139 ++++++- 2 files changed, 456 insertions(+), 34 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index 46f96b751125..1a6ceb843c0f 100644 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -248,6 +248,45 @@ def convert_cast(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_clip(g, op, block): + """Operator converter for clip.""" + + x = g.get_node(op.input("X")[0]) + dtype = infer_type(x).checked_type.dtype + # if the min/max value is a tensor + min_max_is_tensor = False + if op.input("Min"): + min_value = g.get_node(op.input("Min")[0]) + min_value, infered = try_infer_value(min_value, g.get_params()) + if infered: + min_value = min_value.tolist()[0] + if isinstance(min_value, _expr.Expr): + min_max_is_tensor = True + else: + min_value = op.attr("min") + + if op.input("Max"): + max_value = g.get_node(op.input("Max")[0]) + max_value, infered = try_infer_value(max_value, g.get_params()) + if infered: + max_value = max_value.tolist()[0] + if isinstance(max_value, _expr.Expr): + min_max_is_tensor = True + else: + max_value = op.attr("max") + + if min_max_is_tensor: + if not isinstance(min_value, _expr.Expr): + min_value = _op.const(min_value, dtype) + if not isinstance(max_value, _expr.Expr): + max_value = _op.const(max_value, dtype) + out = _op.maximum(x, min_value) + out = _op.minimum(out, max_value) + else: + out = _op.clip(x, min_value, max_value) + g.add_node(op.output("Out")[0], out) + + def convert_concat(g, op, block): """Operator converter for concat.""" @@ -436,6 +475,17 @@ def convert_elementwise_op(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_elu(g, op, block): + """Operator converter for elu.""" + + x = g.get_node(op.input("X")[0]) + dtype = infer_type(x).checked_type.dtype + alpha = op.attr("alpha") + alpha = _expr.const(-1.0 * alpha, dtype=dtype) + out = alpha * _op.nn.relu(_expr.const(1, dtype=dtype) - _op.exp(x)) + _op.nn.relu(x) + g.add_node(op.output("Out")[0], out) + + def convert_expand(g, op, block): """Operator converter for expand.""" @@ -780,6 +830,19 @@ def get_interpolate_mode(op): g.add_node(op.output("Out")[0], out) +def convert_instance_norm(g, op, block): + """Operator converter for instance_norm.""" + + x = g.get_node(op.input("X")[0]) + gamma = g.get_node(op.input("Scale")[0]) + beta = g.get_node(op.input("Bias")[0]) + epsilon = op.attr("epsilon") + + scale = center = True + out = _op.nn.instance_norm(x, gamma, beta, axis=1, epsilon=epsilon, center=center, scale=scale) + g.add_node(op.output("Y")[0], out) + + def convert_layer_norm(g, op, block): """Operator converter for layer_norm.""" @@ -820,6 +883,16 @@ def convert_leaky_relu(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_log1p(g, op, block): + """Operator converter for log1p.""" + + x = g.get_node(op.input("X")[0]) + dtype = infer_type(x).checked_type.dtype + one = _expr.const(1, dtype=dtype) + out = _op.log(x + one) + g.add_node(op.output("Out")[0], out) + + def convert_logical_not(g, op, block): """Operator converter for logical_not op.""" @@ -829,15 +902,65 @@ def convert_logical_not(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_logsigmoid(g, op, block): + """Operator converter for logsigmoid.""" + + x = g.get_node(op.input("X")[0]) + out = _op.log(_op.tensor.sigmoid(x)) + g.add_node(op.output("Out")[0], out) + + +def convert_logsoftmax(g, op, block): + """Operator converter for logsoftmax.""" + + x = g.get_node(op.input("X")[0]) + axis = op.attr("axis") + ndim = len(infer_shape(x)) + if axis < 0: + axis += ndim + m = _op.max(x, [axis], keepdims=True) + e = _op.exp(x - m) + s = _op.sum(e, [axis], keepdims=True) + out = x - m - _op.log(s) + g.add_node(op.output("Out")[0], out) + + +def convert_logsumexp(g, op, block): + """Operator converter for logsumexp.""" + + input_x = g.get_node(op.input("X")[0]) + axis = op.attr("axis") + if op.attr("reduce_all"): + axis = None + keepdims = op.attr("keepdim") + out = get_relay_op("logsumexp")(input_x, axis=axis, keepdims=keepdims) + if not axis and not keepdims: + out = _op.expand_dims(out, axis=0) + g.add_node(op.output("Out")[0], out) + + def convert_lookup_table(g, op, block): """Operator converter for lookup_table_v2.""" indices = g.get_node(op.input("Ids")[0]) padding_idx = op.attr("padding_idx") - if padding_idx != -1: - g.get_params[op.input("W")[0]][padding_idx] = 0.0 - g.add_node(op.input("W")[0], _expr.const(g.params[op.input("W")[0]])) weights = g.get_node(op.input("W")[0]) + if padding_idx != -1: + if op.input("W")[0] in g.get_params(): + weights = g.get_params(op.input("W")[0]) + weights[padding_idx] = 0.0 + weights = _expr.const(weights) + else: + shape, infered = try_infer_value(shape_of(weights), g.get_params()) + if infered: + shape = shape.tolist() + assert not isinstance( + shape, _expr.Expr + ), "Shape of weight has to be fixed for PaddlePaddle's lookup_table" + filters = np.ones(shape).astype(infer_type(weights).checked_type.dtype) + filters[padding_idx] = 0.0 + filters = _expr.const(filters) + weights = weights * filters out = _op.take(weights, indices.astype("int32"), axis=0) g.add_node(op.output("Out")[0], out) @@ -951,6 +1074,16 @@ def flatten_to_nd(x, x_shape, nd=3): g.add_node(op.output("Out")[0], out) +def convert_meshgrid(g, op, block): + """Operator converter for meshgrid.""" + + inputs = op.input("X") + x = [g.get_node(i) for i in inputs] + outs = _op.meshgrid(x, indexing="ij") + for i, out in enumerate(outs): + g.add_node(op.output("Out")[i], out) + + def convert_mul(g, op, block): """Operator converter for mul.""" @@ -996,6 +1129,18 @@ def convert_mul(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_mv(g, op, block): + """Operator converter for mv.""" + + x = g.get_node(op.input("X")[0]) + y = g.get_node(op.input("Vec")[0]) + y = _op.expand_dims(y, axis=-1) + y = _op.transpose(y) + out = _op.nn.dense(x, y) + out = _op.squeeze(out, axis=[-1]) + g.add_node(op.output("Out")[0], out) + + def convert_padding(g, op, block): """Operator converter for padding.""" @@ -1029,6 +1174,15 @@ def convert_padding(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_pixel_shuffle(g, op, block): + """Operator converter for pixel_shuffle.""" + + x = g.get_node(op.input("X")[0]) + upscale_factor = op.attr("upscale_factor") + out = _op.nn.depth_to_space(x, upscale_factor, mode="CRD") + g.add_node(op.output("Out")[0], out) + + def convert_pool2d(g, op, block): """Operator converter for pool2d.""" @@ -1114,6 +1268,49 @@ def convert_pow(g, op, block): g.add_node(op.output("Out")[0], out) +def convert_prelu(g, op, block): + """Operator converter for prelu.""" + + x = g.get_node(op.input("X")[0]) + alpha = g.get_node(op.input("Alpha")[0]) + ndims = len(infer_shape(x)) + axis = 0 if ndims <= 1 else 1 + mode = op.attr("mode") + if mode == "all": + if ndims == 1: + shape = _op.strided_slice(shape_of(x), [0], [1]) + else: + shape = _op.strided_slice(shape_of(x), [1], [2]) + alpha = _op.broadcast_to(alpha, shape) + out = _op.nn.prelu(x, alpha, axis) + g.add_node(op.output("Out")[0], out) + + +def convert_range(g, op, block): + """Operator converter for range.""" + + start = g.get_node(op.input("Start")[0]) + stop = g.get_node(op.input("End")[0]) + step = g.get_node(op.input("Step")[0]) + dtype = infer_type(start).checked_type.dtype + + params = [] + for param in (start, stop, step): + param, infered = try_infer_value(param, g.get_params()) + if infered: + param = param.tolist() + if isinstance(param, list): + param = param[0] + if isinstance(param, _expr.Expr): + param = _op.squeeze(param) + else: + param = _op.const(param, dtype=dtype) + params.append(param) + + out = _op.transform.arange(params[0], params[1], params[2], dtype=dtype) + g.add_node(op.output("Out")[0], out) + + def convert_reciprocal(g, op, block): """Operator converter for reciprocal.""" @@ -1563,38 +1760,119 @@ def convert_size(g, op, block): def convert_slice(g, op, block): """Operator converter for slice.""" - def parameter_process(starts, ends, axes, dshape): - new_axes = [] - new_starts = [] - new_ends = [] - pop_index = 0 - for i in range(max(axes) + 1): - new_axes.append(i) - if i in axes: - new_starts.append(starts[pop_index]) - new_ends.append(ends[pop_index]) - pop_index += 1 - else: - new_starts.append(0) - new_ends.append(dshape[i]) - return new_starts, new_ends, new_axes - data = g.get_node(op.input("Input")[0]) - dshape = infer_shape(data) - starts = op.attr("starts") - ends = op.attr("ends") + dims = len(infer_shape(data)) + axes = op.attr("axes") + indices = _expr.const(axes, dtype="int64") + decrease_axis = op.attr("decrease_axis") - if isinstance(starts, int): - starts = [starts] - if isinstance(ends, int): - ends = [ends] - if isinstance(axes, int): - axes = [axes] if isinstance(decrease_axis, int): decrease_axis = [decrease_axis] - starts, ends, axes = parameter_process(starts, ends, axes, dshape) - out = _op.strided_slice(data, begin=starts, end=ends) + + if op.input("StartsTensor"): + starts = g.get_node(op.input("StartsTensor")[0]) + starts, infered = try_infer_value(starts, g.get_params()) + if infered: + starts = starts.tolist() + elif op.input("StartsTensorList"): + starts = [] + for start_index in op.input("StartsTensorList"): + start_index = g.get_node(start_index).astype("int64") + starts.append(start_index) + starts = _op.concatenate(starts, axis=0) + starts, infered = try_infer_value(starts, g.get_params()) + if infered: + starts = starts.tolist() + else: + starts = op.attr("starts") + + if len(axes) < dims: + if isinstance(starts, _expr.Expr): + starts = _op.scatter( + _op.const([0] * dims, dtype=infer_type(starts).checked_type.dtype), + indices, + starts, + axis=0, + ) + else: + base = [0] * dims + for i, axis in enumerate(axes): + base[axis] = starts[i] + starts = base + + if op.input("EndsTensor"): + ends = g.get_node(op.input("EndsTensor")[0]) + ends, infered = try_infer_value(ends, g.get_params()) + if infered: + ends = ends.tolist() + elif op.input("EndsTensorList"): + ends = [] + for end_index in op.input("EndsTensorList"): + end_index = g.get_node(end_index).astype("int64") + ends.append(end_index) + ends = _op.concatenate(ends, axis=0) + ends, infered = try_infer_value(ends, g.get_params()) + if infered: + ends = ends.tolist() + else: + ends = op.attr("ends") + + if len(axes) < dims: + if isinstance(ends, _expr.Expr): + ends = _op.scatter( + _expr.const( + np.array([np.iinfo(np.int32).max] * dims), + dtype=infer_type(ends).checked_type.dtype, + ), + indices, + ends, + axis=0, + ) + else: + base = [np.iinfo(np.int32).max] * dims + for i, axis in enumerate(axes): + base[axis] = ends[i] + ends = base + + strides = None + if "StridesTensor" in op.input_names and op.input("StridesTensor"): + strides = g.get_node(op.input("StridesTensor")[0]) + strides, infered = try_infer_value(strides, g.get_params()) + if infered: + strides = strides.tolist() + elif "StridesTensorList" in op.input_names and op.input("StridesTensorList"): + strides = [] + for strides_index in op.input("StridesTensorList"): + strides_index = g.get_node(strides_index).astype("int64") + strides.append(strides_index) + strides = _op.concatenate(strides, axis=0) + strides, infered = try_infer_value(strides, g.get_params()) + if infered: + strides = strides.tolist() + elif op.has_attr("strides"): + strides = op.attr("strides") + + if len(axes) < dims: + if isinstance(strides, _expr.Expr): + strides = _op.scatter( + _expr.const( + np.array([1] * dims), + dtype=infer_type(strides).checked_type.dtype, + ), + indices, + strides, + axis=0, + ) + elif strides: + base = [1] * dims + for i, axis in enumerate(axes): + base[axis] = strides[i] + strides = base + if not strides: + strides = _op.const([1] * dims, dtype="int64") + + out = _op.strided_slice(data, begin=starts, end=ends, strides=strides) if decrease_axis: out = _op.squeeze(out, axis=decrease_axis) g.add_node(op.output("Out")[0], out) @@ -1700,6 +1978,7 @@ def convert_unsqueeze(g, op, block): "brelu": convert_brelu, "cast": convert_cast, "ceil": convert_unary_op, + "clip": convert_clip, "concat": convert_concat, "conv2d": convert_conv2d, "conv2d_transpose": convert_conv2d_transpose, @@ -1719,6 +1998,7 @@ def convert_unsqueeze(g, op, block): "elementwise_pow": convert_elementwise_op, "elementwise_prod": convert_elementwise_op, "elementwise_sub": convert_elementwise_op, + "elu": convert_elu, "equal": convert_elementwise_op, "erf": convert_unary_op, "exp": convert_unary_op, @@ -1740,6 +2020,7 @@ def convert_unsqueeze(g, op, block): "hard_shrink": convert_hard_shrink, "hard_sigmoid": convert_hard_sigmoid, "hard_swish": convert_hard_swish, + "instance_norm": convert_instance_norm, "isfinite_v2": convert_unary_op, "isinf_v2": convert_unary_op, "isnan_v2": convert_unary_op, @@ -1750,21 +2031,30 @@ def convert_unsqueeze(g, op, block): "log": convert_unary_op, "log2": convert_unary_op, "log10": convert_unary_op, + "log1p": convert_log1p, "logical_and": convert_binary_logical_op, "logical_not": convert_logical_not, "logical_or": convert_binary_logical_op, "logical_xor": convert_binary_logical_op, + "logsigmoid": convert_logsigmoid, + "log_softmax": convert_logsoftmax, + "logsumexp": convert_logsumexp, "lookup_table_v2": convert_lookup_table, "matmul": convert_matmul, "matmul_v2": convert_matmul, + "meshgrid": convert_meshgrid, "mul": convert_mul, + "mv": convert_mv, "nearest_interp_v2": convert_interpolate, "not_equal": convert_elementwise_op, "pad1d": convert_padding, "pad2d": convert_padding, "pad3d": convert_padding, + "pixel_shuffle": convert_pixel_shuffle, "pool2d": convert_pool2d, "pow": convert_pow, + "prelu": convert_prelu, + "range": convert_range, "relu": convert_unary_op, "relu6": convert_relu6, "reshape2": convert_reshape, @@ -1793,6 +2083,7 @@ def convert_unsqueeze(g, op, block): "softmax": convert_softmax, "softplus": convert_softplus, "softsign": convert_softsign, + "strided_slice": convert_slice, "sqrt": convert_unary_op, "square": convert_square, "squeeze2": convert_squeeze, diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index 9add8ad94206..ea74f949c4e2 100644 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -375,6 +375,37 @@ def forward(self, inputs): verify_model(IsInf(), input_data=input_data) +@tvm.testing.uses_gpu +def test_forward_clip(): + class Clip1(nn.Layer): + @paddle.jit.to_static + def forward(self, inputs): + return paddle.clip(inputs, min=0.3, max=0.55) + + class Clip2(nn.Layer): + @paddle.jit.to_static + def forward(self, inputs, max_value): + return paddle.clip(inputs, max=max_value) + + class Clip3(nn.Layer): + @paddle.jit.to_static + def forward(self, inputs, min_value): + return paddle.clip(inputs, min=min_value) + + class Clip4(nn.Layer): + @paddle.jit.to_static + def forward(self, inputs, min_value, max_value): + return paddle.clip(inputs, min=min_value, max=max_value) + + input_data = paddle.rand((2, 2, 2, 3), dtype="float32") + max_value = paddle.to_tensor([0.55]) + min_value = paddle.to_tensor([0.3]) + verify_model(Clip1(), input_data) + verify_model(Clip2(), [input_data, max_value]) + verify_model(Clip3(), [input_data, min_value]) + verify_model(Clip4(), [input_data, min_value, max_value]) + + @tvm.testing.uses_gpu def test_forward_concat_unsqueeze(): @paddle.jit.to_static @@ -813,6 +844,24 @@ def hard_swish(inputs): verify_model(hard_swish, input_data=input_data) +def test_forward_instance_norm(): + class InstanceNorm(nn.Layer): + def __init__(self, num_features, epsilon=1e-05): + super(InstanceNorm, self).__init__() + self.instance_norm = paddle.nn.InstanceNorm2D( + num_features=num_features, epsilon=epsilon + ) + + def forward(self, inputs): + return self.instance_norm(inputs) + + input_shapes = [[2, 2, 2, 3], [1, 3, 5, 5]] + for input_shape in input_shapes: + input_data = paddle.rand(input_shape, dtype="float32") + verify_model(InstanceNorm(input_shape[1]), input_data) + verify_model(InstanceNorm(input_shape[1], 1e-03), input_data) + + @tvm.testing.uses_gpu def test_forward_interpolate(): class Interpolate(nn.Layer): @@ -1175,6 +1224,9 @@ def forward(self, inputs): verify_model(Reduce("prod", 0), input_data=input_data) verify_model(Reduce("sum", 0, True), input_data=input_data) verify_model(Reduce("mean", -1, True), input_data=input_data) + # logsumexp only supports tensor with rank less than 5 + if len(input_shape) < 5: + verify_model(Reduce("logsumexp", -1, True), input_data=input_data) @tvm.testing.uses_gpu @@ -1256,10 +1308,8 @@ def slice4(inputs): ], ) verify_model(slice2, input_data=input_data) - # need op "strided_slice" - # verify_model(slice3, input_data=paddle.randn((4, 4))) - # need op "assign_value" - # verify_model(slice4, input_data=input_data) + verify_model(slice3, input_data=paddle.randn((4, 4))) + verify_model(slice4, input_data=input_data) @tvm.testing.uses_gpu @@ -1284,14 +1334,18 @@ def forward(self, inputs): "ceil", "cos", "cosh", + "elu", "erf", "exp", "floor", "hardshrink", "hardtanh", + "log_sigmoid", + "log_softmax", "log", "log2", "log10", + "log1p", "reciprocal", "relu", "relu6", @@ -1320,6 +1374,83 @@ def forward(self, inputs): verify_model(MathAPI(api_name), input_data=input_data) +@tvm.testing.uses_gpu +def test_forward_meshgrid(): + @paddle.jit.to_static + def t(x, y, z): + return paddle.meshgrid(x, y, z) + + x = paddle.randint(low=0, high=100, shape=[2]) + y = paddle.randint(low=0, high=100, shape=[3]) + z = paddle.randint(low=0, high=100, shape=[5]) + verify_model(t, [x, y, z]) + + +@tvm.testing.uses_gpu +def test_forward_mv(): + class Mv(nn.Layer): + def forward(self, input1, input2): + return paddle.mv(input1, input2) + + # matrix x vector + input_data1 = paddle.randn((3, 4), dtype="float32") + input_data2 = paddle.randn((4,), dtype="float32") + verify_model(Mv(), input_data=[input_data1, input_data2]) + + +@tvm.testing.uses_gpu +def test_forward_pixel_shuffle(): + class PixelShuffle(nn.Layer): + def __init__(self, upscale_factor): + super(PixelShuffle, self).__init__() + self.pixel_shuffle = paddle.nn.PixelShuffle(upscale_factor) + + @paddle.jit.to_static + def forward(self, x): + return self.pixel_shuffle(x) + + input_shapes = [[1, 4, 3, 3], [2, 8, 2, 5]] + for input_shape in input_shapes: + x = paddle.rand(input_shape, dtype="float32") + verify_model(PixelShuffle(2), x) + + +@tvm.testing.uses_gpu +def test_forward_prelu(): + class PRelu(nn.Layer): + @paddle.jit.to_static + def forward(self, x, w): + return paddle.nn.functional.prelu(x, w) + + x = paddle.normal(shape=[4, 3, 5, 5]) + w = paddle.to_tensor( + np.array( + [ + 0.25, + ] + ).astype("float32") + ) + verify_model(PRelu(), [x, w]) + w2 = paddle.to_tensor(np.array([0.25, 0.5, 0.8]).astype("float32")) + verify_model(PRelu(), [x, w2]) + + +@tvm.testing.uses_gpu +def test_forward_arange(): + @paddle.jit.to_static + def arange(inputs): + return paddle.arange(paddle.shape(inputs)[0], 9, 2.0) + + @paddle.jit.to_static + def arange1(inputs): + return inputs + paddle.arange(0, 10.0, 8, dtype="float32") + + input_shape = [2, 2] + input_data = paddle.rand(input_shape, dtype="float32") + verify_model(arange, input_data) + verify_model(arange1, input_data=input_data) + + @tvm.testing.uses_gpu def test_forward_rnn(): class RNN(nn.Layer):