[ETHOSN] Upgrade NPU driver stack to v22.05 (apache#11759)

* [ETHOSN] Upgrade NPU driver stack to v22.05 In updating the driver stack to v22.05 some additional things needed changes: * Prevent split being offloaded to the NPU which is not supported in v22.05. * Removes compile algorithm configuration option since this was removed in v22.05. Versions before v22.05 will use the default option. * Managing some API changes. * Updating network compile hashes. * Updating expected error message for overall scale bounds check. Change-Id: I09343c398a1f47dec44e135ff8252a6315a9b63f * fix decorator evaluation order Change-Id: Ib1a34093b4011bdc20fca47d474eb1786218de98 * Return none if version doesn't exist For some reason PyTest evaluates the second skipif decorator even if the first one marks the test to be skipped. Thus, meaning test collection fails when we want to check the version. The workaround is to return None when the version is not available. Change-Id: I7cdd8cc70a9ee3c193e9a900f1011829538d975b * Update resnet hash after rebase Change-Id: I7555c4a4d7db4f6c7aa8d476e39277fc5cba2f0d
daobook · Nov 25, 2022 · ad92f24 · ad92f24
1 parent ce7ef12
commit ad92f24
Show file tree

Hide file tree

Showing 12 changed files with 139 additions and 48 deletions.
diff --git a/docker/install/ubuntu_install_ethosn_driver_stack.sh b/docker/install/ubuntu_install_ethosn_driver_stack.sh
@@ -22,7 +22,7 @@ set -o pipefail
 
 repo_url="https://github.com/Arm-software/ethos-n-driver-stack"
 repo_dir="ethosn-driver"
-repo_revision="21.11"
+repo_revision="22.05"
 install_path="/opt/arm/$repo_dir"
 
 tmpdir=$(mktemp -d)

diff --git a/python/tvm/relay/op/contrib/ethosn.py b/python/tvm/relay/op/contrib/ethosn.py
@@ -46,6 +46,11 @@ def ethosn_available():
     return Available.SW_AND_HW if hw else Available.SW_ONLY
 
 
+def ethosn_api_version():
+    """Returns the version of the driver stack api that is being used."""
+    return tvm.get_global_func("relay.ethos-n.api.version")()
+
+
 def partition_for_ethosn(mod, params=None, **opts):
     """Partition the graph greedily offloading supported
     operators to Arm Ethos-N NPU.
@@ -279,6 +284,8 @@ def split(expr):
     """Check if a split is supported by Ethos-N."""
     if not ethosn_available():
         return False
+    if ethosn_api_version() == 2205:
+        return False
     if not support.split(expr):
         return False
 

diff --git a/src/relay/backend/contrib/ethosn/codegen.cc b/src/relay/backend/contrib/ethosn/codegen.cc
@@ -662,8 +662,6 @@ sl::CompilationOptions EthosnCompiler::CreateOptions() {
   options.m_EnableIntermediateCompression = cfg.value()->enable_intermediate_compression;
   options.m_DisableWinograd = cfg.value()->disable_winograd;
   options.m_DebugInfo.m_DebugDir = cfg.value()->debug_dir;
-  options.m_CompilerAlgorithm =
-      sl::EthosNCompilerAlgorithmFromString(cfg.value()->compiler_algorithm.c_str());
   return options;
 }
 

diff --git a/src/relay/backend/contrib/ethosn/codegen_ethosn.h b/src/relay/backend/contrib/ethosn/codegen_ethosn.h
@@ -248,7 +248,6 @@ struct EthosnCompilerConfigNode : public tvm::AttrsNode<EthosnCompilerConfigNode
   bool enable_intermediate_compression;
   bool disable_winograd;
   String debug_dir;
-  String compiler_algorithm;
 
   TVM_DECLARE_ATTRS(EthosnCompilerConfigNode, "ext.attrs.EthosnCompilerConfigNode") {
     TVM_ATTR_FIELD(variant).describe("See Ethos-N documentation.").set_default("n78");
@@ -276,7 +275,6 @@ struct EthosnCompilerConfigNode : public tvm::AttrsNode<EthosnCompilerConfigNode
     TVM_ATTR_FIELD(enable_intermediate_compression).set_default(true);
     TVM_ATTR_FIELD(disable_winograd).set_default(false);
     TVM_ATTR_FIELD(debug_dir).set_default(".");
-    TVM_ATTR_FIELD(compiler_algorithm).set_default("NonCascadingOnly");
   }
 };
 

diff --git a/src/relay/backend/contrib/ethosn/ethosn_api_version.h b/src/relay/backend/contrib/ethosn/ethosn_api_version.h
@@ -35,6 +35,9 @@
  */
 
 #if ETHOSN_SUPPORT_LIBRARY_VERSION_MAJOR == 3 && ETHOSN_SUPPORT_LIBRARY_VERSION_MINOR == 0 && \
+    ETHOSN_SUPPORT_LIBRARY_VERSION_PATCH == 1
+#define _ETHOSN_API_VERSION_ 2205
+#elif ETHOSN_SUPPORT_LIBRARY_VERSION_MAJOR == 3 && ETHOSN_SUPPORT_LIBRARY_VERSION_MINOR == 0 && \
     ETHOSN_SUPPORT_LIBRARY_VERSION_PATCH == 0
 #define _ETHOSN_API_VERSION_ 2111
 #else

diff --git a/src/runtime/contrib/ethosn/ethosn_device.cc b/src/runtime/contrib/ethosn/ethosn_device.cc
@@ -163,12 +163,21 @@ bool Inference(tvm::runtime::TVMArgs args, dl::Network* npu,
       case 8: {
         dl::Buffer** ofms = &ofm_raw[0];
         for (DLTensor* tensor : outputs) {
-          uint8_t* source_buffer_data = (*ofms++)->GetMappedBuffer();
+          dl::Buffer* source_buffer = (*ofms++);
+#if _ETHOSN_API_VERSION_ < 2111
+          uint8_t* source_buffer_data = source_buffer->GetMappedBuffer();
+#else
+          uint8_t* source_buffer_data = source_buffer->Map();
+#endif
           uint8_t* dest_pointer = static_cast<uint8_t*>(tensor->data);
           if (source_buffer_data != dest_pointer) {
             CopyOutput<uint8_t>(ofm_raw, &outputs);
             break;
           }
+#if _ETHOSN_API_VERSION_ >= 2111
+          source_buffer->Unmap();
+#else
+#endif
         }
         break;
       }

diff --git a/tests/python/contrib/test_ethosn/infrastructure.py b/tests/python/contrib/test_ethosn/infrastructure.py
@@ -270,6 +270,33 @@ def test_error(mod, params, err_msg):
     assert err_msg in caught, caught
 
 
+def get_overall_scale_range_expected_error_message():
+    """
+    Get the expected error message when the overall scale is out of range.
+    Different versions of the driver stack support different scale ranges,
+    so creating a unified utility to obtain the expected message.
+    """
+    if get_ethosn_api_version() >= 2205:
+        lb = "2^-32"
+    elif get_ethosn_api_version() > 2102:
+        lb = "2.328306e-10"
+    else:
+        lb = "0"
+
+    if get_ethosn_api_version() >= 2205:
+        ub = "65536"
+    else:
+        ub = "1"
+
+    lb_bracket = "(" if get_ethosn_api_version() >= 2205 else "["
+    ub_bracket = ")"
+
+    return (
+        "Overall scale (of the input * weights / output) should be in the range "
+        f"{lb_bracket}{lb}, {ub}{ub_bracket}"
+    )
+
+
 def get_conv2d(var, shape, dtype):
     """Standard convolution to test activation functions"""
 
@@ -333,7 +360,8 @@ def get_conv2d_qnn_params(
 
 
 def get_ethosn_api_version():
-    return tvm.get_global_func("relay.ethos-n.api.version")()
+    gf = tvm.get_global_func("relay.ethos-n.api.version", True)
+    return gf() if gf else None
 
 
 def get_ethosn_variant():

diff --git a/tests/python/contrib/test_ethosn/test_conv2d.py b/tests/python/contrib/test_ethosn/test_conv2d.py
@@ -208,16 +208,15 @@ def test_conv2d(dtype, depthwise):
 
 @requires_ethosn
 def test_conv2d_failure():
-    lb = "2.328306e-10" if tei.get_ethosn_api_version() > 2102 else "0"
     trials = [
         (
             (1, 4, 4, 4),
             1,
             1,
             0,
-            1,
+            1024,
             0,
-            1,
+            1024,
             0,
             1,
             "none",
@@ -227,7 +226,7 @@ def test_conv2d_failure():
             "uint8",
             8,
             "HWIO",
-            f"Overall scale (of the input * weights / output) should be in the range [{lb}, 1)",
+            tei.get_overall_scale_range_expected_error_message(),
         ),
         (
             (1, 4, 4, 4),

diff --git a/tests/python/contrib/test_ethosn/test_fullyconnected.py b/tests/python/contrib/test_ethosn/test_fullyconnected.py
@@ -111,19 +111,18 @@ def test_fullyconnected(dtype):
 
 @requires_ethosn
 def test_fullyconnected_failure():
-    lb = "2.328306e-10" if tei.get_ethosn_api_version() > 2102 else "0"
     trials = [
         (
             (1, 64),
             (1, 64),
             0,
-            1,
+            1024,
             0,
-            1,
+            1024,
             0,
             1,
             "uint8",
-            f"Overall scale (of the input * weights / output) should be in the range [{lb}, 1)",
+            tei.get_overall_scale_range_expected_error_message(),
         ),
         (
             (1, 1, 1, 64),

diff --git a/tests/python/contrib/test_ethosn/test_networks.py b/tests/python/contrib/test_ethosn/test_networks.py
@@ -118,18 +118,21 @@ def get_model():
 
 @requires_ethosn
 def test_mobilenet_v1():
-    # If this test is failing due to a hash mismatch, please notify @mbaret and
+    # If this test is failing due to a hash mismatch, please notify @lhutton1 and
     # @Leo-arm. The hash is there to catch any changes in the behaviour of the
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    _compile_hash = {"393a19dfb980345cdd3bbeddbc36424d"}
-    if tei.get_ethosn_api_version() == 2111:
+    if tei.get_ethosn_api_version() == 2205:
+        _compile_hash = {"cb12b5469d78af81f4704488e3857755"}
+    elif tei.get_ethosn_api_version() == 2111:
         _compile_hash = {"5d1c6a6bd4df8963866cc90405bf92dd"}
-    if tei.get_ethosn_api_version() == 2102:
+    elif tei.get_ethosn_api_version() == 2102:
         _compile_hash = {"46ccafc840633633aca441645e41b444"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
             _compile_hash = {"e4ed29dceb1187505948ab17fc3cc6d6"}
+    else:
+        _compile_hash = {"393a19dfb980345cdd3bbeddbc36424d"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
@@ -145,40 +148,44 @@ def test_mobilenet_v1():
 
 @requires_ethosn
 def test_resnet_50_int8():
-    # If this test is failing due to a hash mismatch, please notify @mbaret and
+    # If this test is failing due to a hash mismatch, please notify @lhutton1 and
     # @Leo-arm. The hash is there to catch any changes in the behaviour of the
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    if tei.get_ethosn_api_version() > 2011:
-        if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
-            _compile_hash = {"c0a01c547ed1b2e3308094508fa1bfea", "434f0c65c41e24d5482142c88b3438fe"}
-            _test_image_network(
-                model_url="https://raw.githubusercontent.com/dmlc/web-data/main/tensorflow/"
-                "models/Quantized/resnet_50_quantized.tflite",
-                model_sub_path="resnet_50_quantized.tflite",
-                input_dict={"input": (1, 224, 224, 3)},
-                compile_hash=_compile_hash,
-                output_count=1,
-                host_ops=11,
-                npu_partitions=2,
-            )
+    if tei.get_ethosn_api_version() == 2205:
+        _compile_hash = {"c0a01c547ed1b2e3308094508fa1bfea", "64905a4ff2dbde08078ccc9f44ad711d"}
+    else:
+        _compile_hash = {"c0a01c547ed1b2e3308094508fa1bfea", "434f0c65c41e24d5482142c88b3438fe"}
+    _test_image_network(
+        model_url="https://raw.githubusercontent.com/dmlc/web-data/main/tensorflow/"
+        "models/Quantized/resnet_50_quantized.tflite",
+        model_sub_path="resnet_50_quantized.tflite",
+        input_dict={"input": (1, 224, 224, 3)},
+        compile_hash=_compile_hash,
+        output_count=1,
+        host_ops=11,
+        npu_partitions=2,
+    )
 
 
 @requires_ethosn
 def test_inception_v3():
-    # If this test is failing due to a hash mismatch, please notify @mbaret and
+    # If this test is failing due to a hash mismatch, please notify @lhutton1 and
     # @Leo-arm. The hash is there to catch any changes in the behaviour of the
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    _compile_hash = {"2c7ff5487e1a21e62b3b42eec624fed4"}
-    if tei.get_ethosn_api_version() == 2111:
+    if tei.get_ethosn_api_version() == 2205:
+        _compile_hash = {"85ef702ad3628c598db8c72060c70a61"}
+    elif tei.get_ethosn_api_version() == 2111:
         _compile_hash = {"e6abe33a7bc4a4170da53eefa6577bba"}
-    if tei.get_ethosn_api_version() == 2102:
+    elif tei.get_ethosn_api_version() == 2102:
         _compile_hash = {"43dc2097127eb224c0191b1a15f8acca"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
             _compile_hash = {"7db23387bdc5af6eaa1ae3f7d456caf0"}
+    else:
+        _compile_hash = {"2c7ff5487e1a21e62b3b42eec624fed4"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite_11_05_08/inception_v3_quant.tgz",
@@ -193,18 +200,21 @@ def test_inception_v3():
 
 @requires_ethosn
 def test_inception_v4():
-    # If this test is failing due to a hash mismatch, please notify @mbaret and
+    # If this test is failing due to a hash mismatch, please notify @lhutton1 and
     # @Leo-arm. The hash is there to catch any changes in the behaviour of the
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    _compile_hash = {"4245dbd02e1432dc261a67fc8e632a00"}
-    if tei.get_ethosn_api_version() == 2111:
+    if tei.get_ethosn_api_version() == 2205:
+        _compile_hash = {"91a980eaf53881f4f109a1a7578e422b"}
+    elif tei.get_ethosn_api_version() == 2111:
         _compile_hash = {"42e43c323ed8202f7b720ba9029bbcb7"}
-    if tei.get_ethosn_api_version() == 2102:
+    elif tei.get_ethosn_api_version() == 2102:
         _compile_hash = {"fab6c2297502f95d33079c6ce1a737f9"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
             _compile_hash = {"8da68849b75613ac3dffd3fff2dd87da"}
+    else:
+        _compile_hash = {"4245dbd02e1432dc261a67fc8e632a00"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/inception_v4_299_quant_20181026.tgz",
@@ -219,19 +229,21 @@ def test_inception_v4():
 
 @requires_ethosn
 def test_ssd_mobilenet_v1():
-    # If this test is failing due to a hash mismatch, please notify @mbaret and
+    # If this test is failing due to a hash mismatch, please notify @lhutton1 and
     # @Leo-arm. The hash is there to catch any changes in the behaviour of the
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
-    _compile_hash = {"5ee8ed6af9a7f31fc14957b51a8e7423", "e6a91ccc47ba4c6b4614fcd676bd726f"}
-    if tei.get_ethosn_api_version() == 2111:
-        # TODO(Leo-arm): review split operator
+    if tei.get_ethosn_api_version() == 2205:
+        _compile_hash = {"d804ce3496a776c48f719b4062d5e5c3", "afb68ca8f452d1f4a674b457b5e30f59"}
+    elif tei.get_ethosn_api_version() == 2111:
         _compile_hash = {"a37f900601b9493bd142e8aed16205e5", "afb68ca8f452d1f4a674b457b5e30f59"}
-    if tei.get_ethosn_api_version() == 2102:
+    elif tei.get_ethosn_api_version() == 2102:
         _compile_hash = {"7795b6c67178da9d1f9b98063bad75b1", "10826406ae724e52f360a06c35ced09d"}
         if tei.get_ethosn_variant() == "Ethos-N78_1TOPS_2PLE_RATIO":
             _compile_hash = {"928dc6ae5ce49a4ad63ca87f7575970f", "b092f9820f7e9341fc53daa781b98772"}
+    else:
+        _compile_hash = {"5ee8ed6af9a7f31fc14957b51a8e7423", "e6a91ccc47ba4c6b4614fcd676bd726f"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip",

diff --git a/tests/python/contrib/test_ethosn/test_split.py b/tests/python/contrib/test_ethosn/test_split.py
@@ -31,6 +31,10 @@ def _get_model(shape, dtype, splits, axis):
     return split.astuple()
 
 
+@pytest.mark.skipif(
+    tei.get_ethosn_api_version() == 2205,
+    reason="Split is not supported by the 22.05 release of the driver stack",
+)
 @requires_ethosn
 @pytest.mark.parametrize("dtype", ["uint8", "int8"])
 def test_split(dtype):
@@ -60,6 +64,10 @@ def test_split(dtype):
         tei.verify(outputs, dtype, 0)
 
 
+@pytest.mark.skipif(
+    tei.get_ethosn_api_version() == 2205,
+    reason="Split is not supported by the 22.05 release of the driver stack",
+)
 @requires_ethosn
 def test_split_failure():
     trials = [

diff --git a/tests/python/contrib/test_ethosn/test_topologies.py b/tests/python/contrib/test_ethosn/test_topologies.py
@@ -73,7 +73,20 @@ def get_model(input_shape, dtype, var_names):
     for npu in [False, True]:
         model = get_model(inputs["a"].shape, dtype, iter(inputs))
         mod = tei.make_module(model, [])
-        outputs.append(tei.build_and_run(mod, inputs, 1, {}, npu=npu))
+
+        expected_host_ops = 1 if tei.get_ethosn_api_version() == 2205 else 0
+        npu_partitions = 2 if tei.get_ethosn_api_version() == 2205 else 1
+        outputs.append(
+            tei.build_and_run(
+                mod,
+                inputs,
+                1,
+                {},
+                npu=npu,
+                expected_host_ops=expected_host_ops,
+                npu_partitions=npu_partitions,
+            )
+        )
 
     tei.verify(outputs, dtype, 2)
 
@@ -202,11 +215,28 @@ def get_model(shape, dtype, splits, axis):
         for npu in [False, True]:
             model = get_model(shape, dtype, splits, axis)
             mod = tei.make_module(model, {})
-            outputs.append(tei.build_and_run(mod, inputs, 2, {}, npu=npu))
+
+            expected_host_ops = 1 if tei.get_ethosn_api_version() == 2205 else 0
+            npu_partitions = 2 if tei.get_ethosn_api_version() == 2205 else 1
+            outputs.append(
+                tei.build_and_run(
+                    mod,
+                    inputs,
+                    2,
+                    {},
+                    npu=npu,
+                    expected_host_ops=expected_host_ops,
+                    npu_partitions=npu_partitions,
+                )
+            )
 
         tei.verify(outputs, dtype, 0)
 
 
+@pytest.mark.skipif(
+    tei.get_ethosn_api_version() == 2205,
+    reason="Split is not supported by the 22.05 release of the driver stack",
+)
 @requires_ethosn
 @pytest.mark.parametrize("dtype", ["uint8", "int8"])
 def test_output_tuple_propagation(dtype):