From 9541391935d26492414101e0a1800fc1691d5040 Mon Sep 17 00:00:00 2001 From: Aleksei-grovety <113356454+Aleksei-grovety@users.noreply.github.com> Date: Fri, 9 Jun 2023 16:44:59 +0400 Subject: [PATCH 1/4] [microNPU][ETHOSU] Fix Softmax activation parameters Fix activation parameters for operations according to the values in Vela. --- .../contrib/ethosu/softmax_rewriter.py | 71 +++++++++++++------ 1 file changed, 48 insertions(+), 23 deletions(-) diff --git a/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py b/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py index 6c0a1dffc30c..72044ba48d42 100644 --- a/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py +++ b/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py @@ -82,6 +82,8 @@ def callback( self, pre: tvm.relay.Expr, post: tvm.relay.Expr, node_map: tvm.ir.container.Map ) -> tvm.relay.Expr: params = self.params_class(post.op.body) + quant_min = -128 + quant_max = 127 ifm = post.args[0] ifm_dtype = ifm.checked_type.dtype @@ -121,12 +123,14 @@ def callback( ifm2_scale=0.0, ifm2_zero_point=int(params.ifm.q_params.zero_point), ofm_scale=1.0, - ofm_zero_point=127, + ofm_zero_point=quant_max, ifm_channels=depth, ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", activation="LUT", + clip_min=quant_min - int(params.ifm.q_params.zero_point), + clip_max=quant_max - int(params.ifm.q_params.zero_point), ) # PASS 2 - SHR @@ -147,8 +151,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, rounding_mode="NATURAL", ) @@ -165,6 +169,9 @@ def callback( ofm_channels=1, upscale="NONE", ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 4 - CLZ @@ -177,6 +184,9 @@ def callback( ofm_scale=0.0, ofm_zero_point=int(params.ifm.q_params.zero_point), ofm_channels=1, + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 5 - Sub @@ -196,6 +206,9 @@ def callback( ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 6 - Sub @@ -215,6 +228,9 @@ def callback( ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 7 - SHL @@ -234,8 +250,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 8 - Sub @@ -255,6 +271,9 @@ def callback( ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 9 - SHL @@ -274,8 +293,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 10 - Add @@ -296,8 +315,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, use_rescale=True, rescale_scale=1, rescale_shift=1, @@ -321,8 +340,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128 * 2, - clip_max=127 * 2, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 12 - Add @@ -343,8 +362,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, ) nr_x = rescale_w_offset @@ -368,8 +387,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128 * 2, - clip_max=127 * 2, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 14, 19, 24 - Sub @@ -388,6 +407,9 @@ def callback( ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 15, 20, 25 - Mul @@ -407,8 +429,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128 * 2, - clip_max=127 * 2, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 16, 21, 26 - Mul @@ -428,8 +450,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 17, 22, 27 - Add @@ -448,6 +470,9 @@ def callback( ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", + activation="CLIP", + clip_min=quant_min, + clip_max=quant_max, ) # PASS 28 - Mul @@ -468,8 +493,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128, - clip_max=127, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 29 - Mul @@ -489,8 +514,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="CLIP", - clip_min=-128 * 2, - clip_max=127 * 2, + clip_min=quant_min, + clip_max=quant_max, ) # PASS 30 - SHR From cf800c6730b3bf4eb87a3fed5b6e70608662d7be Mon Sep 17 00:00:00 2001 From: Aleksei-grovety <113356454+Aleksei-grovety@users.noreply.github.com> Date: Tue, 20 Jun 2023 14:41:04 +0400 Subject: [PATCH 2/4] fix legalization parameters --- .../tvm/relay/backend/contrib/ethosu/softmax_rewriter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py b/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py index 72044ba48d42..23d4f4b45b11 100644 --- a/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py +++ b/python/tvm/relay/backend/contrib/ethosu/softmax_rewriter.py @@ -129,8 +129,8 @@ def callback( reversed_operands=False, ofm_dtype="int32", activation="LUT", - clip_min=quant_min - int(params.ifm.q_params.zero_point), - clip_max=quant_max - int(params.ifm.q_params.zero_point), + clip_min=-255, + clip_max=0, ) # PASS 2 - SHR @@ -245,7 +245,7 @@ def callback( ifm2_zero_point=0, ofm_scale=0.0, ofm_zero_point=int(params.ifm.q_params.zero_point), - ifm_channels=depth, + ifm_channels=1, ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", @@ -335,7 +335,7 @@ def callback( ifm2_zero_point=0, ofm_scale=2.0, ofm_zero_point=0, - ifm_channels=depth, + ifm_channels=1, ifm2_channels=1, reversed_operands=False, ofm_dtype="int32", From 17150f70d41ee0ed8ece1f0d1432fedbdaa5aa20 Mon Sep 17 00:00:00 2001 From: Aleksei-grovety <113356454+Aleksei-grovety@users.noreply.github.com> Date: Tue, 20 Jun 2023 14:41:47 +0400 Subject: [PATCH 3/4] Update test_legalize.py --- .../contrib/test_ethosu/test_legalize.py | 141 ++++++++---------- 1 file changed, 63 insertions(+), 78 deletions(-) diff --git a/tests/python/contrib/test_ethosu/test_legalize.py b/tests/python/contrib/test_ethosu/test_legalize.py index 1b643f815721..a77621d45f8e 100644 --- a/tests/python/contrib/test_ethosu/test_legalize.py +++ b/tests/python/contrib/test_ethosu/test_legalize.py @@ -269,6 +269,7 @@ def tf_function(self, input_shape): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -387,6 +388,7 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -524,6 +526,7 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -540,7 +543,6 @@ def representative_dataset(): return tflite_model def verify(ext_func): - assert ArePadOnGraph().are_pad_on_graph(ext_func.body) == True conv2d_pattern_table = [ @@ -718,6 +720,7 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -818,6 +821,7 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -944,6 +948,7 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -960,7 +965,6 @@ def representative_dataset(): return tflite_model def verify(ext_func, channel_padding): - op = ext_func.body pad_before = 0 @@ -1163,7 +1167,6 @@ def representative_dataset(): return tflite_model def expected_mod(): - expected_ir_string = "" if activation_function == "NONE" and pooling_type == "AVG": @@ -1638,7 +1641,6 @@ def verify(ext_func): ], ) def test_relay_reshape_legalize(ifm_shape, new_shape): - ifm = relay.var("ifm", shape=ifm_shape, dtype="int8") reshape = relay.op.reshape(ifm, new_shape) func = relay.Function([ifm], reshape) @@ -3302,7 +3304,6 @@ def verify(ext_func): legalize.NoOpRewriter(), ] for legalizer in seq: - mod["tvmgen_default_ethos_u_main_0"] = dataflow_pattern.rewrite( legalizer, mod["tvmgen_default_ethos_u_main_0"] ) @@ -3401,6 +3402,7 @@ def fully_connected(self, x): concrete_func = model.fully_connected.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) + # Convert the model def representative_dataset(): for _ in range(100): @@ -3526,9 +3528,60 @@ def representative_dataset(): assert tuple(func_body.args[1].checked_type.shape) == (256,) -@pytest.mark.parametrize("ifm_shape", [(1, 12), (1, 12, 32)]) -def test_tflite_softmax(ifm_shape): +def test_tflite_softmax(): + np.random.seed(0) dtype = "int8" + ifm_shape = (1, 12) + + def get_expected_mod(): + # fmt: off + mod_str = """ + #[version = "0.0.5"] + def @main(%x: Tensor[(1, 12), int8] /* ty=Tensor[(1, 12), int8] span=x:0:0 */, output_tensor_names=["Identity"]) -> Tensor[(1, 12), int8] { + @tvmgen_default_ethos_u_main_0(%x) /* ty=Tensor[(1, 12), int8] */ + } + + def @tvmgen_default_ethos_u_main_0(%y: Tensor[(1, 12), int8] /* ty=Tensor[(1, 12), int8] */, Compiler="ethos-u", Primitive=1, Inline=1, global_symbol="tvmgen_default_ethos_u_main_0") -> Tensor[(1, 12), int8] { + %0 = reshape(%y, newshape=[1, 1, 12, 1]); + %1 = reshape(%0, newshape=[1, 1, 1, 12]); + %2 = contrib.ethosu.pooling(%0, meta[relay.Constant][0], pooling_type="MAX", ifm_scale=0.011756030842661858f, ifm_zero_point=-43, ofm_scale=0f, ofm_zero_point=-43, pool_shape=[1, 12], ofm_channels=1, ofm_dtype="int8"); + %3 = contrib.ethosu.binary_elementwise(%1, %2, meta[relay.Constant][1], operator_type="SUB", ifm_scale=0.011756030842661858f, ifm_zero_point=-43, ifm2_scale=0f, ifm2_zero_point=-43, ofm_scale=1f, ofm_zero_point=127, ifm_channels=12, ifm2_channels=1, activation="LUT", clip_min=-255, clip_max=0, ofm_dtype="int32"); + %4 = contrib.ethosu.binary_elementwise(%3, meta[relay.Constant][2], meta[relay.Constant][0], operator_type="SHR", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=12, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, rounding_mode="NATURAL", ofm_dtype="int32"); + %5 = contrib.ethosu.pooling(%4, meta[relay.Constant][0], pooling_type="SUM", ifm_scale=0f, ifm_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, pool_shape=[1, 1], ofm_channels=1, ofm_dtype="int32", activation="CLIP", clip_min=-128, clip_max=127); + %6 = contrib.ethosu.unary_elementwise(%5, meta[relay.Constant][0], operator_type="CLZ", ifm_scale=0f, ifm_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ofm_channels=1, activation="CLIP", clip_min=-128, clip_max=127); + %7 = contrib.ethosu.binary_elementwise(%6, meta[relay.Constant][3], meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %8 = contrib.ethosu.binary_elementwise(%5, %7, meta[relay.Constant][0], operator_type="SHL", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %9 = contrib.ethosu.binary_elementwise(%8, meta[relay.Constant][4], meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %10 = contrib.ethosu.binary_elementwise(%9, meta[relay.Constant][3], meta[relay.Constant][0], operator_type="SHL", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %11 = contrib.ethosu.binary_elementwise(%10, meta[relay.Constant][5], meta[relay.Constant][0], operator_type="ADD", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32", use_rescale=True, rescale_scale=1, rescale_shift=1); + %12 = contrib.ethosu.binary_elementwise(%11, meta[relay.Constant][6], meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %13 = contrib.ethosu.binary_elementwise(%12, meta[relay.Constant][7], meta[relay.Constant][0], operator_type="ADD", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %14 = contrib.ethosu.binary_elementwise(%13, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %15 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %14, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %16 = contrib.ethosu.binary_elementwise(%13, %15, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %17 = contrib.ethosu.binary_elementwise(%16, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %18 = contrib.ethosu.binary_elementwise(%13, %17, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %19 = contrib.ethosu.binary_elementwise(%18, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %20 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %19, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %21 = contrib.ethosu.binary_elementwise(%18, %20, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %22 = contrib.ethosu.binary_elementwise(%21, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %23 = contrib.ethosu.binary_elementwise(%18, %22, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %24 = contrib.ethosu.binary_elementwise(%23, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %25 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %24, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %26 = contrib.ethosu.binary_elementwise(%23, %25, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %27 = contrib.ethosu.binary_elementwise(%26, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %28 = contrib.ethosu.binary_elementwise(%23, %27, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %29 = contrib.ethosu.binary_elementwise(%28, meta[relay.Constant][10], meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %30 = contrib.ethosu.binary_elementwise(%3, %29, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=12, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %31 = contrib.ethosu.binary_elementwise(meta[relay.Constant][11], %6, meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); + %32 = contrib.ethosu.binary_elementwise(%30, %31, meta[relay.Constant][0], operator_type="SHR", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0.00390625f, ofm_zero_point=-128, ifm_channels=12, ifm2_channels=1, rounding_mode="NATURAL", ofm_dtype="int8"); + reshape(%32, newshape=[1, 12]) + } + + #[metadata]{ "root": 1, "nodes": [ { "type_key": "" }, { "type_key": "Map", "keys": [ "relay.Constant" ], "data": [2] }, { "type_key": "Array", "data": [ 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ] }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "0", "span": "0", "virtual_device_": "4" } }, { "type_key": "VirtualDevice", "attrs": { "device_type_int": "-1", "memory_scope": "5", "target": "0", "virtual_device_id": "-1" } }, { "type_key": "runtime.String" }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "1", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "2", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "3", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "4", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "5", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "6", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "7", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "8", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "9", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "10", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "11", "span": "0", "virtual_device_": "4" } } ], "b64ndarrays": [ "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAAAQAAAAAgAQAAAAAAAAAAAAAAAAAAAAAA", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAAAQAAAAAgAQAAAQAAAAAAAAAEAAAAAAAAOAljBu1edgYu74kGq7qdBhbCsQYmBsYGkIfaBg5H7wZcRQQHOIMZB2IBLwecwEQHq8FaB1gFcQdrjIcHsVeeB/hntQcRvswHzVrkBwI//AeHaxQIN+EsCDehRQjiq14IYAJ4CJWlkQhplqsIxtXFCJZk4AjPQ/sIYnQWCUX3MQlxzU0J4vdpCZZ3hgmSTaMJ1XrACW8A3gln3/sJyxgaCq2tOAofn1cKOe52CnCclgo2qrYKDhnXChnq9wp+HhkLa7c6Cw+2XAucG38LS+mhC1IgxQv5wegLfc8MDCZKMQw9M1YMEox7DPZVoQw+kscMQkLuDGVnFQ0FAz0NhxZlDcijjQ1Zq7YNKC/gDaowCg5WsTQOs7JfDkM2iw6OPbcOI8rjDpXdEA98eT4Pd59sDyFRmw8rkMoPPF76DwW9KhA8rlsQmjONEN9OvxDQAfIQMU4lEW02WRE0u40RBN/CEbqj+BE8Cy8ScxdmElLKnRLMJdYS5ysPE6PeSBMLQIMTL1K+EyYX+hMPkTYUC8JzFECssRTpUfAUNrUvFWbYbxW5vbAVemfyFfrXNBZQEngWcRe8Fo7qABcNjkYXYwSNFwtQ1BeJcxwYbHFlGEdMrxiwBvoYWqNFGekkkhkUjt8ZmOEtGjkifRrFUs0aEHYeG/KOcBtdoMMbOK0XHHi4bBwSxsIcQtcZHRbwcR2mE8sdEEUlHpGHgB5a3twerkw6H9vVmB84ffgfKEZZIBc0uyB0Sh4h0YyCIbL+5yGro04iX3+2IneVHyOp6YkjtX/1I1tbYiS6gdAkVvQ/JWq4sCXt0SIm5kSWJmQVCyeJR4Endt/4J3rhcSjTUewo0zRoKeCO5SlnZGQq5LnkKuKTZivu9ukrweduLARr9Sx0hX0t3zsHLh6Tki6tkR8vbTmuLxyRPjDjndAw3WRkMT7r+TFNNpEyY0sqM+cvxTNV6WE0KH0ANRnxoDXISkM28o/nNmfGjTcI9DU4xh7gOKVMjDmpgzo6FMrqOg8mnTvgnVE88DkIPeb8wD1O73s+vRc5P858+D9gJbpAOxh+QUtcREKR+AxDKPTXQztWpUQQJnVF8GpHRm4sHEf9cfNHO0PNSNinqUmep4hKakpqSzCYTkzmmDVNbVcfTg3XC0/KIvtPEkPtUGtA4lF5I9pS+/TUU7y90lTahtNVWVnXVmY+3ldQP+hYgWX1WX26BVvlRxlcYRcwXfoySl6NpGdfLnaIYAuyrGFvYtRiEpX/YyFOLmW/m2Bm6IiWZ2sg0GhWbQ1q33pOa1lUk2xBBdxtN5kob+IbeXBgmc1xpB0mc9y0gnRca+N1nk1Idz5osXj9xx56oXmQe3KKBn19B4F+////fw==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAADAAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAQAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAQA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAA////fw==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAxMPDww==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAWlpaWg==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAIA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAABAAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAgAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAIwAAAA==" ]} + """ + # fmt: on + return tvm.relay.fromtext(mod_str) def create_tflite_graph(): @tf.function @@ -3536,10 +3589,11 @@ def softmax(x): return tf.nn.softmax(x) concrete_func = softmax.get_concrete_function(tf.TensorSpec(ifm_shape, dtype=tf.float32)) + # Convert the model def representative_dataset(): for _ in range(100): - data = np.random.rand(*tuple(ifm_shape)) + data = np.random.uniform(low=-1, high=2, size=tuple(ifm_shape)) yield [data.astype(np.float32)] converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func]) @@ -3551,75 +3605,6 @@ def representative_dataset(): tflite_model = converter.convert() return tflite_model - def verify(ext_func): - out_op = ext_func.body - ops = [] - # List of expected operations and their type if it exists - expected_ops = [ - ("reshape", None), - ("reshape", None), - ("contrib.ethosu.pooling", "MAX"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "SHR"), - ("contrib.ethosu.pooling", "SUM"), - ("contrib.ethosu.unary_elementwise", "CLZ"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "SHL"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "SHL"), - ("contrib.ethosu.binary_elementwise", "ADD"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "ADD"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "ADD"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "ADD"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "ADD"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "MUL"), - ("contrib.ethosu.binary_elementwise", "SUB"), - ("contrib.ethosu.binary_elementwise", "SHR"), - ("reshape", None), - ] - - def get_op_type(op): - if hasattr(op.attrs, "pooling_type"): - return op.attrs.pooling_type - elif hasattr(op.attrs, "operator_type"): - return op.attrs.operator_type - return None - - def _visit(stmt): - if isinstance(stmt, relay.expr.Call): - ops.append(stmt) - - relay.analysis.post_order_visit(out_op, _visit) - - # check IFM - ifm = ops[0].args[0].checked_type - assert list(ifm.shape) == list(ifm_shape) - assert str(ifm.dtype) == dtype - - # check OFM - ofm = out_op.checked_type - assert list(ofm.shape) == list(ifm_shape) - assert ofm.dtype == dtype - - # check operations - - ops = [(op.op.name, get_op_type(op)) for op in ops] - assert expected_ops == ops - softmax_pattern_table = [ ( ethosu.SoftMaxParams.composite_name, @@ -3643,7 +3628,7 @@ def _visit(stmt): ) mod = relay.transform.InferType()(mod) - verify(mod["tvmgen_default_ethos_u_main_0"]) + tvm.ir.assert_structural_equal(mod, get_expected_mod()) if __name__ == "__main__": From d55ebb0076966be21e4492cf9893852be33249e2 Mon Sep 17 00:00:00 2001 From: Aleksei-grovety <113356454+Aleksei-grovety@users.noreply.github.com> Date: Wed, 21 Jun 2023 10:48:08 +0400 Subject: [PATCH 4/4] Update test_legalize.py --- .../contrib/test_ethosu/test_legalize.py | 163 +++++++++++------- 1 file changed, 104 insertions(+), 59 deletions(-) diff --git a/tests/python/contrib/test_ethosu/test_legalize.py b/tests/python/contrib/test_ethosu/test_legalize.py index a77621d45f8e..c952a13c52d6 100644 --- a/tests/python/contrib/test_ethosu/test_legalize.py +++ b/tests/python/contrib/test_ethosu/test_legalize.py @@ -269,7 +269,6 @@ def tf_function(self, input_shape): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -388,7 +387,6 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -526,7 +524,6 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -543,6 +540,7 @@ def representative_dataset(): return tflite_model def verify(ext_func): + assert ArePadOnGraph().are_pad_on_graph(ext_func.body) == True conv2d_pattern_table = [ @@ -720,7 +718,6 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -821,7 +818,6 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -948,7 +944,6 @@ def tf_function(self, x): concrete_func = model.tf_function.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -965,6 +960,7 @@ def representative_dataset(): return tflite_model def verify(ext_func, channel_padding): + op = ext_func.body pad_before = 0 @@ -1167,6 +1163,7 @@ def representative_dataset(): return tflite_model def expected_mod(): + expected_ir_string = "" if activation_function == "NONE" and pooling_type == "AVG": @@ -1641,6 +1638,7 @@ def verify(ext_func): ], ) def test_relay_reshape_legalize(ifm_shape, new_shape): + ifm = relay.var("ifm", shape=ifm_shape, dtype="int8") reshape = relay.op.reshape(ifm, new_shape) func = relay.Function([ifm], reshape) @@ -3304,6 +3302,7 @@ def verify(ext_func): legalize.NoOpRewriter(), ] for legalizer in seq: + mod["tvmgen_default_ethos_u_main_0"] = dataflow_pattern.rewrite( legalizer, mod["tvmgen_default_ethos_u_main_0"] ) @@ -3402,7 +3401,6 @@ def fully_connected(self, x): concrete_func = model.fully_connected.get_concrete_function( tf.TensorSpec(ifm_shape, dtype=tf.float32) ) - # Convert the model def representative_dataset(): for _ in range(100): @@ -3533,63 +3531,12 @@ def test_tflite_softmax(): dtype = "int8" ifm_shape = (1, 12) - def get_expected_mod(): - # fmt: off - mod_str = """ - #[version = "0.0.5"] - def @main(%x: Tensor[(1, 12), int8] /* ty=Tensor[(1, 12), int8] span=x:0:0 */, output_tensor_names=["Identity"]) -> Tensor[(1, 12), int8] { - @tvmgen_default_ethos_u_main_0(%x) /* ty=Tensor[(1, 12), int8] */ - } - - def @tvmgen_default_ethos_u_main_0(%y: Tensor[(1, 12), int8] /* ty=Tensor[(1, 12), int8] */, Compiler="ethos-u", Primitive=1, Inline=1, global_symbol="tvmgen_default_ethos_u_main_0") -> Tensor[(1, 12), int8] { - %0 = reshape(%y, newshape=[1, 1, 12, 1]); - %1 = reshape(%0, newshape=[1, 1, 1, 12]); - %2 = contrib.ethosu.pooling(%0, meta[relay.Constant][0], pooling_type="MAX", ifm_scale=0.011756030842661858f, ifm_zero_point=-43, ofm_scale=0f, ofm_zero_point=-43, pool_shape=[1, 12], ofm_channels=1, ofm_dtype="int8"); - %3 = contrib.ethosu.binary_elementwise(%1, %2, meta[relay.Constant][1], operator_type="SUB", ifm_scale=0.011756030842661858f, ifm_zero_point=-43, ifm2_scale=0f, ifm2_zero_point=-43, ofm_scale=1f, ofm_zero_point=127, ifm_channels=12, ifm2_channels=1, activation="LUT", clip_min=-255, clip_max=0, ofm_dtype="int32"); - %4 = contrib.ethosu.binary_elementwise(%3, meta[relay.Constant][2], meta[relay.Constant][0], operator_type="SHR", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=12, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, rounding_mode="NATURAL", ofm_dtype="int32"); - %5 = contrib.ethosu.pooling(%4, meta[relay.Constant][0], pooling_type="SUM", ifm_scale=0f, ifm_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, pool_shape=[1, 1], ofm_channels=1, ofm_dtype="int32", activation="CLIP", clip_min=-128, clip_max=127); - %6 = contrib.ethosu.unary_elementwise(%5, meta[relay.Constant][0], operator_type="CLZ", ifm_scale=0f, ifm_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ofm_channels=1, activation="CLIP", clip_min=-128, clip_max=127); - %7 = contrib.ethosu.binary_elementwise(%6, meta[relay.Constant][3], meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %8 = contrib.ethosu.binary_elementwise(%5, %7, meta[relay.Constant][0], operator_type="SHL", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %9 = contrib.ethosu.binary_elementwise(%8, meta[relay.Constant][4], meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %10 = contrib.ethosu.binary_elementwise(%9, meta[relay.Constant][3], meta[relay.Constant][0], operator_type="SHL", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %11 = contrib.ethosu.binary_elementwise(%10, meta[relay.Constant][5], meta[relay.Constant][0], operator_type="ADD", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32", use_rescale=True, rescale_scale=1, rescale_shift=1); - %12 = contrib.ethosu.binary_elementwise(%11, meta[relay.Constant][6], meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %13 = contrib.ethosu.binary_elementwise(%12, meta[relay.Constant][7], meta[relay.Constant][0], operator_type="ADD", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %14 = contrib.ethosu.binary_elementwise(%13, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %15 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %14, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %16 = contrib.ethosu.binary_elementwise(%13, %15, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %17 = contrib.ethosu.binary_elementwise(%16, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %18 = contrib.ethosu.binary_elementwise(%13, %17, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %19 = contrib.ethosu.binary_elementwise(%18, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %20 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %19, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %21 = contrib.ethosu.binary_elementwise(%18, %20, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %22 = contrib.ethosu.binary_elementwise(%21, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %23 = contrib.ethosu.binary_elementwise(%18, %22, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %24 = contrib.ethosu.binary_elementwise(%23, %11, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %25 = contrib.ethosu.binary_elementwise(meta[relay.Constant][8], %24, meta[relay.Constant][0], operator_type="SUB", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %26 = contrib.ethosu.binary_elementwise(%23, %25, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %27 = contrib.ethosu.binary_elementwise(%26, meta[relay.Constant][9], meta[relay.Constant][0], operator_type="MUL", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %28 = contrib.ethosu.binary_elementwise(%23, %27, meta[relay.Constant][0], operator_type="ADD", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %29 = contrib.ethosu.binary_elementwise(%28, meta[relay.Constant][10], meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=1f, ofm_zero_point=0, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %30 = contrib.ethosu.binary_elementwise(%3, %29, meta[relay.Constant][0], operator_type="MUL", ifm_scale=1f, ifm_zero_point=0, ifm2_scale=1f, ifm2_zero_point=0, ofm_scale=2f, ofm_zero_point=0, ifm_channels=12, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %31 = contrib.ethosu.binary_elementwise(meta[relay.Constant][11], %6, meta[relay.Constant][0], operator_type="SUB", ifm_scale=0f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0f, ofm_zero_point=-43, ifm_channels=1, ifm2_channels=1, activation="CLIP", clip_min=-128, clip_max=127, ofm_dtype="int32"); - %32 = contrib.ethosu.binary_elementwise(%30, %31, meta[relay.Constant][0], operator_type="SHR", ifm_scale=2f, ifm_zero_point=0, ifm2_scale=0f, ifm2_zero_point=0, ofm_scale=0.00390625f, ofm_zero_point=-128, ifm_channels=12, ifm2_channels=1, rounding_mode="NATURAL", ofm_dtype="int8"); - reshape(%32, newshape=[1, 12]) - } - - #[metadata]{ "root": 1, "nodes": [ { "type_key": "" }, { "type_key": "Map", "keys": [ "relay.Constant" ], "data": [2] }, { "type_key": "Array", "data": [ 3, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 ] }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "0", "span": "0", "virtual_device_": "4" } }, { "type_key": "VirtualDevice", "attrs": { "device_type_int": "-1", "memory_scope": "5", "target": "0", "virtual_device_id": "-1" } }, { "type_key": "runtime.String" }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "1", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "2", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "3", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "4", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "5", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "6", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "7", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "8", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "9", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "10", "span": "0", "virtual_device_": "4" } }, { "type_key": "relay.Constant", "attrs": { "_checked_type_": "0", "data": "11", "span": "0", "virtual_device_": "4" } } ], "b64ndarrays": [ "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAAAQAAAAAgAQAAAAAAAAAAAAAAAAAAAAAA", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAAAQAAAAAgAQAAAQAAAAAAAAAEAAAAAAAAOAljBu1edgYu74kGq7qdBhbCsQYmBsYGkIfaBg5H7wZcRQQHOIMZB2IBLwecwEQHq8FaB1gFcQdrjIcHsVeeB/hntQcRvswHzVrkBwI//AeHaxQIN+EsCDehRQjiq14IYAJ4CJWlkQhplqsIxtXFCJZk4AjPQ/sIYnQWCUX3MQlxzU0J4vdpCZZ3hgmSTaMJ1XrACW8A3gln3/sJyxgaCq2tOAofn1cKOe52CnCclgo2qrYKDhnXChnq9wp+HhkLa7c6Cw+2XAucG38LS+mhC1IgxQv5wegLfc8MDCZKMQw9M1YMEox7DPZVoQw+kscMQkLuDGVnFQ0FAz0NhxZlDcijjQ1Zq7YNKC/gDaowCg5WsTQOs7JfDkM2iw6OPbcOI8rjDpXdEA98eT4Pd59sDyFRmw8rkMoPPF76DwW9KhA8rlsQmjONEN9OvxDQAfIQMU4lEW02WRE0u40RBN/CEbqj+BE8Cy8ScxdmElLKnRLMJdYS5ysPE6PeSBMLQIMTL1K+EyYX+hMPkTYUC8JzFECssRTpUfAUNrUvFWbYbxW5vbAVemfyFfrXNBZQEngWcRe8Fo7qABcNjkYXYwSNFwtQ1BeJcxwYbHFlGEdMrxiwBvoYWqNFGekkkhkUjt8ZmOEtGjkifRrFUs0aEHYeG/KOcBtdoMMbOK0XHHi4bBwSxsIcQtcZHRbwcR2mE8sdEEUlHpGHgB5a3twerkw6H9vVmB84ffgfKEZZIBc0uyB0Sh4h0YyCIbL+5yGro04iX3+2IneVHyOp6YkjtX/1I1tbYiS6gdAkVvQ/JWq4sCXt0SIm5kSWJmQVCyeJR4Endt/4J3rhcSjTUewo0zRoKeCO5SlnZGQq5LnkKuKTZivu9ukrweduLARr9Sx0hX0t3zsHLh6Tki6tkR8vbTmuLxyRPjDjndAw3WRkMT7r+TFNNpEyY0sqM+cvxTNV6WE0KH0ANRnxoDXISkM28o/nNmfGjTcI9DU4xh7gOKVMjDmpgzo6FMrqOg8mnTvgnVE88DkIPeb8wD1O73s+vRc5P858+D9gJbpAOxh+QUtcREKR+AxDKPTXQztWpUQQJnVF8GpHRm4sHEf9cfNHO0PNSNinqUmep4hKakpqSzCYTkzmmDVNbVcfTg3XC0/KIvtPEkPtUGtA4lF5I9pS+/TUU7y90lTahtNVWVnXVmY+3ldQP+hYgWX1WX26BVvlRxlcYRcwXfoySl6NpGdfLnaIYAuyrGFvYtRiEpX/YyFOLmW/m2Bm6IiWZ2sg0GhWbQ1q33pOa1lUk2xBBdxtN5kob+IbeXBgmc1xpB0mc9y0gnRca+N1nk1Idz5osXj9xx56oXmQe3KKBn19B4F+////fw==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAADAAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAQAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAQA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAA////fw==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAxMPDww==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAWlpaWg==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAAAAIA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAABAAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAAgAAAA==", "P6G0lvBAXt0AAAAAAAAAAAEAAAAAAAAABAAAAAAgAQABAAAAAAAAAAEAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAQAAAAAAAAAIwAAAA==" ]} - """ - # fmt: on - return tvm.relay.fromtext(mod_str) - def create_tflite_graph(): @tf.function def softmax(x): return tf.nn.softmax(x) concrete_func = softmax.get_concrete_function(tf.TensorSpec(ifm_shape, dtype=tf.float32)) - # Convert the model def representative_dataset(): for _ in range(100): @@ -3605,6 +3552,104 @@ def representative_dataset(): tflite_model = converter.convert() return tflite_model + def verify(ext_func): + out_op = ext_func.body + ops = [] + # List of expected operations, their type and activation parameters if it exists + expected_ops_params = [ + ("reshape", None, [None, None, None, None, None, None]), + ("reshape", None, [None, None, None, None, None, None]), + ("contrib.ethosu.pooling", "MAX", [0.011756093241274357, -43, None, None, 0.0, -43]), + ( + "contrib.ethosu.binary_elementwise", + "SUB", + [0.011756093241274357, -43, 0.0, -43, 1.0, 127], + ), + ("contrib.ethosu.binary_elementwise", "SHR", [1.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.pooling", "SUM", [0.0, 0, None, None, 0.0, -43]), + ("contrib.ethosu.unary_elementwise", "CLZ", [0.0, 0, None, None, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "SUB", [0.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "SHL", [0.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "SUB", [0.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "SHL", [0.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "ADD", [0.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "ADD", [2.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "SUB", [2.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [2.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "ADD", [1.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "SUB", [2.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [2.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "ADD", [1.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "SUB", [2.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [2.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "ADD", [1.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 0.0, 0, 1.0, 0]), + ("contrib.ethosu.binary_elementwise", "MUL", [1.0, 0, 1.0, 0, 2.0, 0]), + ("contrib.ethosu.binary_elementwise", "SUB", [0.0, 0, 0.0, 0, 0.0, -43]), + ("contrib.ethosu.binary_elementwise", "SHR", [2.0, 0, 0.0, 0, 0.00390625, -128]), + ("reshape", None, [None, None, None, None, None, None]), + ] + + def get_attr_value(op, attr_name): + if hasattr(op.attrs, attr_name): + return op.attrs[attr_name] + else: + return None + + def get_op_type(op): + if hasattr(op.attrs, "pooling_type"): + return op.attrs.pooling_type + elif hasattr(op.attrs, "operator_type"): + return op.attrs.operator_type + return None + + def get_activation_params(op): + activation_params = [] + activation_params.append(get_attr_value(op, "ifm_scale")) + activation_params.append(get_attr_value(op, "ifm_zero_point")) + activation_params.append(get_attr_value(op, "ifm2_scale")) + activation_params.append(get_attr_value(op, "ifm2_zero_point")) + activation_params.append(get_attr_value(op, "ofm_scale")) + activation_params.append(get_attr_value(op, "ofm_zero_point")) + return activation_params + + def _visit(stmt): + if isinstance(stmt, relay.expr.Call): + ops.append(stmt) + + relay.analysis.post_order_visit(out_op, _visit) + + # check IFM + ifm = ops[0].args[0].checked_type + assert list(ifm.shape) == list(ifm_shape) + assert str(ifm.dtype) == dtype + + # check OFM + ofm = out_op.checked_type + assert list(ofm.shape) == list(ifm_shape) + assert ofm.dtype == dtype + + # check operations + for op, expected_op_params in zip(ops, expected_ops_params): + activation_params = get_activation_params(op) + expected_op_name, expected_op_type, expected_activation_params = expected_op_params + assert op.op.name == expected_op_name + assert expected_op_type == get_op_type(op) + for activation_param, expected_activation_param in zip( + activation_params, expected_activation_params + ): + if isinstance(activation_param, float): + assert math.isclose(expected_activation_param, activation_param, abs_tol=1e-7) + else: + assert expected_activation_param == activation_param + softmax_pattern_table = [ ( ethosu.SoftMaxParams.composite_name, @@ -3628,7 +3673,7 @@ def representative_dataset(): ) mod = relay.transform.InferType()(mod) - tvm.ir.assert_structural_equal(mod, get_expected_mod()) + verify(mod["tvmgen_default_ethos_u_main_0"]) if __name__ == "__main__":