From e4844fe3264d7e657b3103fda4607fe6d3ba50e0 Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous Date: Tue, 11 Apr 2023 15:24:11 +0800 Subject: [PATCH 01/10] add attr support for paddle ops --- python/tvm/relay/frontend/paddlepaddle.py | 26 ++++++++++++++----- .../frontend/paddlepaddle/test_forward.py | 19 ++++++++++++-- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index 70015b497eb4..4a229cbedfcb 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -502,7 +502,9 @@ def convert_dropout(g, op, block): """Operator converter for dropout.""" x = g.get_node(op.input("X")[0]) - g.add_node(op.output("Out")[0], x) + dropout_prob = op.attr("dropout_prob") + out = _op.nn.dropout(x, dropout_prob) + g.add_node(op.output("Out")[0], out) def convert_dot(g, op, block): @@ -827,10 +829,17 @@ def convert_gelu(g, op, block): """Operator converter for gelu.""" x = g.get_node(op.input("X")[0]) - out = x * ( - _expr.const(0.5, dtype="float32") - + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32") - ) + approximate = op.attr("approximate") + if approximate: + out = x * ( + _expr.const(0.5, dtype="float32") + + (_op.tanh((_expr.const(0.7978846, dtype="float32")*(x+_expr.const(0.044715, dtype="float32")*x*x*x)))) * _expr.const(0.5, dtype="float32") + ) + else: + out = x * ( + _expr.const(0.5, dtype="float32") + + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32") + ) g.add_node(op.output("Out")[0], out) @@ -897,8 +906,9 @@ def convert_hard_sigmoid(g, op, block): """Operator converter for hard_sigmoid.""" slope = op.attr("slope") + offset = op.attr("offset") x = g.get_node(op.input("X")[0]) - out = x * _expr.const(slope) + _expr.const(0.5) + out = x * _expr.const(slope) + _expr.const(offset) out = _op.clip(out, 0, 1) g.add_node(op.output("Out")[0], out) @@ -1425,7 +1435,9 @@ def convert_pixel_shuffle(g, op, block): x = g.get_node(op.input("X")[0]) upscale_factor = op.attr("upscale_factor") - out = _op.nn.depth_to_space(x, upscale_factor, mode="CRD") + data_format = op.attr("data_format") + warnings.warn(str(op.attr("data_format"))) + out = _op.nn.depth_to_space(x, block_size=upscale_factor, layout=data_format, mode="CRD") g.add_node(op.output("Out")[0], out) diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index 5bdbd68842fb..0df28cfbab3f 100755 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -616,11 +616,15 @@ def test_forward_dropout(): @paddle.jit.to_static def dropout(inputs): return nn.functional.dropout(inputs) + def dropout1(inputs): + return nn.functional.dropout(inputs, 0.1) input_shape = [1, 3, 10, 10] input_data = paddle.rand(input_shape, dtype="float32") verify_model(dropout, input_data=input_data[0, 0]) verify_model(dropout, input_data=input_data) + verify_model(dropout1, input_data=input_data[0, 0]) + verify_model(dropout1, input_data=input_data) def test_forward_elemwise(): @@ -996,10 +1000,13 @@ def test_forward_gelu(): @paddle.jit.to_static def gelu(inputs): return nn.functional.gelu(inputs) + def gelu1(inputs): + return nn.functional.gelu(inputs, approximate=True) input_shape = [1, 3, 10, 10] input_data = paddle.rand(input_shape, dtype="float32") verify_model(gelu, input_data=input_data) + verify_model(gelu1, input_data=input_data) @tvm.testing.uses_gpu @@ -1007,10 +1014,13 @@ def test_forward_hard_sigmoid(): @paddle.jit.to_static def hard_sigmoid(inputs): return nn.functional.hardsigmoid(inputs) + def hard_sigmoid1(inputs): + return nn.functional.hardsigmoid(inputs, offset=0.6) input_shape = [1, 3, 10, 10] input_data = paddle.rand(input_shape, dtype="float32") verify_model(hard_sigmoid, input_data=input_data) + verify_model(hard_sigmoid1, input_data=input_data) @tvm.testing.uses_gpu @@ -1781,9 +1791,9 @@ def forward(self, input1, input2): @tvm.testing.uses_gpu def test_forward_pixel_shuffle(): class PixelShuffle(nn.Layer): - def __init__(self, upscale_factor): + def __init__(self, upscale_factor, data_format="NCHW"): super(PixelShuffle, self).__init__() - self.pixel_shuffle = paddle.nn.PixelShuffle(upscale_factor) + self.pixel_shuffle = paddle.nn.PixelShuffle(upscale_factor, data_format) @paddle.jit.to_static def forward(self, x): @@ -1794,6 +1804,11 @@ def forward(self, x): x = paddle.rand(input_shape, dtype="float32") verify_model(PixelShuffle(2), x) + input_shapes = [[1, 3, 3, 4], [2, 2, 5, 8]] + for input_shape in input_shapes: + x = paddle.rand(input_shape, dtype="float32") + verify_model(PixelShuffle(2, data_format="NHWC"), x) + @tvm.testing.uses_gpu def test_forward_prelu(): From 93486001e9b72a5dc25c00affca5603fdf2bd6ac Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:21:21 +0800 Subject: [PATCH 02/10] Update paddlepaddle.py --- python/tvm/relay/frontend/paddlepaddle.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index 4a229cbedfcb..b6c83476d658 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -833,12 +833,20 @@ def convert_gelu(g, op, block): if approximate: out = x * ( _expr.const(0.5, dtype="float32") - + (_op.tanh((_expr.const(0.7978846, dtype="float32")*(x+_expr.const(0.044715, dtype="float32")*x*x*x)))) * _expr.const(0.5, dtype="float32") + + ( + _op.tanh( + ( + _expr.const(0.7978846, dtype="float32") + *(x + _expr.const(0.044715, dtype="float32")*x*x*x) + ) + ) + ) * _expr.const(0.5, dtype="float32") ) - else: + else: out = x * ( _expr.const(0.5, dtype="float32") - + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32") + + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) + * _expr.const(0.5, dtype="float32") ) g.add_node(op.output("Out")[0], out) From 571c8cc1cf25597bb6b25e539e50903affc398aa Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:21:54 +0800 Subject: [PATCH 03/10] Update test_forward.py --- tests/python/frontend/paddlepaddle/test_forward.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index 0df28cfbab3f..def845143391 100755 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -616,6 +616,7 @@ def test_forward_dropout(): @paddle.jit.to_static def dropout(inputs): return nn.functional.dropout(inputs) + def dropout1(inputs): return nn.functional.dropout(inputs, 0.1) @@ -1000,6 +1001,7 @@ def test_forward_gelu(): @paddle.jit.to_static def gelu(inputs): return nn.functional.gelu(inputs) + def gelu1(inputs): return nn.functional.gelu(inputs, approximate=True) @@ -1014,6 +1016,7 @@ def test_forward_hard_sigmoid(): @paddle.jit.to_static def hard_sigmoid(inputs): return nn.functional.hardsigmoid(inputs) + def hard_sigmoid1(inputs): return nn.functional.hardsigmoid(inputs, offset=0.6) From 052ddf5db222a23a197e6f914fa5aa46396d17fd Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:28:55 +0800 Subject: [PATCH 04/10] codestyle --- python/tvm/relay/frontend/paddlepaddle.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index b6c83476d658..0ae6f1cdbae5 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -837,15 +837,16 @@ def convert_gelu(g, op, block): _op.tanh( ( _expr.const(0.7978846, dtype="float32") - *(x + _expr.const(0.044715, dtype="float32")*x*x*x) + * (x + _expr.const(0.044715, dtype="float32") * x * x * x) ) ) - ) * _expr.const(0.5, dtype="float32") + ) + * _expr.const(0.5, dtype="float32") ) else: out = x * ( _expr.const(0.5, dtype="float32") - + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) + + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32") ) g.add_node(op.output("Out")[0], out) From a42162aa235781c88bf6206c8b29764e24e10cc2 Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 11:08:10 +0800 Subject: [PATCH 05/10] remove approximate strategy in gelu --- python/tvm/relay/frontend/paddlepaddle.py | 26 +++++------------------ 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index 0ae6f1cdbae5..4b9c34718beb 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -829,26 +829,11 @@ def convert_gelu(g, op, block): """Operator converter for gelu.""" x = g.get_node(op.input("X")[0]) - approximate = op.attr("approximate") - if approximate: - out = x * ( - _expr.const(0.5, dtype="float32") - + ( - _op.tanh( - ( - _expr.const(0.7978846, dtype="float32") - * (x + _expr.const(0.044715, dtype="float32") * x * x * x) - ) - ) - ) - * _expr.const(0.5, dtype="float32") - ) - else: - out = x * ( - _expr.const(0.5, dtype="float32") - + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) - * _expr.const(0.5, dtype="float32") - ) + out = x * ( + _expr.const(0.5, dtype="float32") + + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) + * _expr.const(0.5, dtype="float32") + ) g.add_node(op.output("Out")[0], out) @@ -1445,7 +1430,6 @@ def convert_pixel_shuffle(g, op, block): x = g.get_node(op.input("X")[0]) upscale_factor = op.attr("upscale_factor") data_format = op.attr("data_format") - warnings.warn(str(op.attr("data_format"))) out = _op.nn.depth_to_space(x, block_size=upscale_factor, layout=data_format, mode="CRD") g.add_node(op.output("Out")[0], out) From f47e7e98cc455bada1e83a23dcb011aa5701e9de Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 11:19:59 +0800 Subject: [PATCH 06/10] remove gelu test --- tests/python/frontend/paddlepaddle/test_forward.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index def845143391..c54022ed8cb1 100755 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -1002,13 +1002,9 @@ def test_forward_gelu(): def gelu(inputs): return nn.functional.gelu(inputs) - def gelu1(inputs): - return nn.functional.gelu(inputs, approximate=True) - input_shape = [1, 3, 10, 10] input_data = paddle.rand(input_shape, dtype="float32") verify_model(gelu, input_data=input_data) - verify_model(gelu1, input_data=input_data) @tvm.testing.uses_gpu From bfc3a9cbf857e28bc5b1bce3216d07e3a085f1c5 Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 11:37:57 +0800 Subject: [PATCH 07/10] codestyle --- python/tvm/relay/frontend/paddlepaddle.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index 4b9c34718beb..ad8e8fe3cbcf 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -831,8 +831,7 @@ def convert_gelu(g, op, block): x = g.get_node(op.input("X")[0]) out = x * ( _expr.const(0.5, dtype="float32") - + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) - * _expr.const(0.5, dtype="float32") + + _op.erf(x * _expr.const(0.5**0.5, dtype="float32")) * _expr.const(0.5, dtype="float32") ) g.add_node(op.output("Out")[0], out) From 6a38cc88b861fcb498ad0dba0cae84be93691a30 Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 14:21:47 +0800 Subject: [PATCH 08/10] add dropout_implementation attr for dropout --- python/tvm/relay/frontend/paddlepaddle.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/tvm/relay/frontend/paddlepaddle.py b/python/tvm/relay/frontend/paddlepaddle.py index ad8e8fe3cbcf..8a92d50522af 100755 --- a/python/tvm/relay/frontend/paddlepaddle.py +++ b/python/tvm/relay/frontend/paddlepaddle.py @@ -503,7 +503,11 @@ def convert_dropout(g, op, block): x = g.get_node(op.input("X")[0]) dropout_prob = op.attr("dropout_prob") - out = _op.nn.dropout(x, dropout_prob) + dropout_implementation = op.attr("dropout_implementation") + if dropout_implementation == "downgrade_in_infer": + out = _op.nn.dropout(x, dropout_prob) * _expr.const(1 - dropout_prob, dtype="float32") + else: + out = _op.nn.dropout(x, dropout_prob) g.add_node(op.output("Out")[0], out) From b99deadf4c3f469bdac9e44b42e28843a981a8c8 Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 14:22:25 +0800 Subject: [PATCH 09/10] add tests for dropout --- tests/python/frontend/paddlepaddle/test_forward.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index c54022ed8cb1..04c8d3d8da82 100755 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -616,9 +616,14 @@ def test_forward_dropout(): @paddle.jit.to_static def dropout(inputs): return nn.functional.dropout(inputs) - + + @paddle.jit.to_static def dropout1(inputs): return nn.functional.dropout(inputs, 0.1) + + @paddle.jit.to_static + def dropout2(inputs): + return nn.functional.dropout(inputs, 0.1, mode="downscale_in_infer") input_shape = [1, 3, 10, 10] input_data = paddle.rand(input_shape, dtype="float32") @@ -626,6 +631,8 @@ def dropout1(inputs): verify_model(dropout, input_data=input_data) verify_model(dropout1, input_data=input_data[0, 0]) verify_model(dropout1, input_data=input_data) + verify_model(dropout2, input_data=input_data[0, 0]) + verify_model(dropout2, input_data=input_data) def test_forward_elemwise(): From abc3746f8f3c7dd1762254f2385807cba821a0fa Mon Sep 17 00:00:00 2001 From: MayYouBeProsperous <99723454+MayYouBeProsperous@users.noreply.github.com> Date: Wed, 12 Apr 2023 15:25:22 +0800 Subject: [PATCH 10/10] codestyle --- tests/python/frontend/paddlepaddle/test_forward.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/frontend/paddlepaddle/test_forward.py b/tests/python/frontend/paddlepaddle/test_forward.py index 04c8d3d8da82..289fc0faa3ef 100755 --- a/tests/python/frontend/paddlepaddle/test_forward.py +++ b/tests/python/frontend/paddlepaddle/test_forward.py @@ -616,11 +616,11 @@ def test_forward_dropout(): @paddle.jit.to_static def dropout(inputs): return nn.functional.dropout(inputs) - + @paddle.jit.to_static def dropout1(inputs): return nn.functional.dropout(inputs, 0.1) - + @paddle.jit.to_static def dropout2(inputs): return nn.functional.dropout(inputs, 0.1, mode="downscale_in_infer")