* CLML test cases incmpliance with pytest infra

python/tvm/testing/utils.py

@@ -933,6 +933,15 @@ def _any_gpu_exists():
     parent_features="gpu",
 )
 
+# Mark a test as requiring OpenCLML support in build.
+requires_openclml = Feature(
+    "OpenCLML",
+    "CLML",
+    cmake_flag="USE_CLML",
+    target_kind_enabled="opencl",
+)
+
+
 # Mark a test as requiring microTVM to run
 requires_micro = Feature("micro", "MicroTVM", cmake_flag="USE_MICRO")
 

tests/python/contrib/test_clml/conftest.py

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import sys
+import tvm
+import pytest
+from test_clml.infrastructure import Device
+
+
+@pytest.fixture(scope="session")
+def device():
+    return Device()

tests/python/contrib/test_clml/infrastructure.py

@@ -73,12 +73,12 @@ class Device:
     """
 
     connection_type = "tracker"
-    host = "localhost"
-    port = 9150
+    host = os.getenv("TVM_TRACKER_HOST", "localhost")
+    port = int(os.getenv("TVM_TRACKER_PORT", 9090))
     target = "opencl"
     target_host = "llvm -mtriple=aarch64-linux-gnu"
     device_key = "android"
-    cross_compile = "aarch64-linux-android-g++"
+    cross_compile = os.getenv("TVM_NDK_CC", "aarch64-linux-android-g++")
 
     def __init__(self):
         """Keep remote device for lifetime of object."""
@@ -100,43 +100,6 @@ def _get_remote(cls):
 
         return device
 
-    @classmethod
-    def load(cls, file_name):
-        """Load test config
-
-        Load the test configuration by looking for file_name relative
-        to the test_clml directory.
-        """
-        location = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
-        config_file = os.path.join(location, file_name)
-        if not os.path.exists(config_file):
-            warnings.warn("Config file doesn't exist, resuming CLML tests with default config.")
-            return
-        with open(config_file, mode="r") as config:
-            test_config = json.load(config)
-
-        cls.connection_type = test_config["connection_type"]
-        cls.host = test_config["host"]
-        cls.port = test_config["port"]
-        cls.target = test_config["target"]
-        cls.target_host = test_config["target_host"]
-        cls.device_key = test_config.get("device_key") or ""
-        cls.cross_compile = test_config.get("cross_compile") or ""
-
-
-def skip_runtime_test():
-    """Skip test if it requires the runtime and it's not present."""
-    # CLML codegen not present.
-    if not tvm.get_global_func("relay.ext.clml", True):
-        print("Skip because CLML codegen is not available.")
-        return True
-
-    # Remote device is in use or CLML runtime not present
-    # Note: Ensure that the device config has been loaded before this check
-    if not Device.connection_type != "local" and not clml.is_clml_runtime_enabled():
-        print("Skip because runtime isn't present or a remote device isn't being used.")
-        return True
-
 
 def skip_codegen_test():
     """Skip test if it requires the CLML codegen and it's not present."""

tests/python/contrib/test_clml/test_network.py

@@ -16,13 +16,13 @@
 # under the License.
 """OpenCL ML network tests."""
 
+import tvm
 import numpy as np
-import pytest
-from tvm import testing
 from tvm import relay
-
-import tvm
-from test_clml.infrastructure import skip_runtime_test, build_and_run, Device
+from tvm.relay import testing
+from tvm.contrib import utils
+from test_clml.infrastructure import build_and_run, Device
+import pytest
 
 
 def _build_and_run_network(mod, params, inputs, data, device, atol, rtol, tvm_log=""):
@@ -59,15 +59,9 @@ def get_bottom_top_model(model, layer_name):
     return mod, params, ref_output
 
 
-def test_mobilenet():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    dtype = "float16"
-
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def test_mobilenet(device, dtype):
     def get_model():
         from tensorflow.keras.applications import MobileNet
         import tensorflow as tf
@@ -107,15 +101,9 @@ def get_model():
     tvm.testing.assert_allclose(opencl_sort[:10], clml_sort[:10], rtol=1e-5, atol=1e-5)
 
 
-def test_inception_v3():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    dtype = "float16"
-
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def test_inception_v3(device, dtype):
     def get_model():
         from tensorflow.keras.applications import InceptionV3
         import tensorflow as tf
@@ -150,15 +138,9 @@ def get_model():
     tvm.testing.assert_allclose(opencl_sort[:5], clml_sort[:5], rtol=1e-5, atol=1e-5)
 
 
-def test_resnet50v2():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    dtype = "float16"
-
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def test_resnet50v2(device, dtype):
     def get_model():
         from tensorflow.keras.applications import ResNet50V2
         import tensorflow as tf
@@ -202,9 +184,3 @@ def get_model():
     clml_sort = np.argsort(outputs[0].asnumpy()).flatten()
 
     tvm.testing.assert_allclose(opencl_sort[:10], clml_sort[:10], rtol=1e-5, atol=1e-5)
-
-
-if __name__ == "__main__":
-    test_mobilenet()
-    test_resnet50v2()
-    test_inception_v3()

tests/python/contrib/test_clml/test_ops.py

@@ -16,21 +16,14 @@
 # under the License.
 """CLML integration conv2d tests."""
 
-import numpy as np
-
-np.random.seed(0)
-
 import tvm
-from tvm import testing
+import numpy as np
 from tvm import relay
+from tvm.relay import testing
 from tvm.ir import IRModule
-
-from test_clml.infrastructure import (
-    skip_runtime_test,
-    skip_codegen_test,
-    build_and_run,
-    Device,
-)
+from tvm.contrib import utils
+from test_clml.infrastructure import build_and_run, Device, skip_codegen_test
+import pytest
 
 
 def _get_conv_model(
@@ -98,17 +91,9 @@ def _get_conv_model(
     return out, params
 
 
-def test_conv2d():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    np.random.seed(0)
-
-    dtype = "float32"
-
+@pytest.mark.parametrize("dtype", ["float32"])
+@tvm.testing.requires_openclml
+def test_conv2d(device, dtype):
     trials = [
         # Normal convolution
         [3, 3, (1, 1), (1, 1), (1, 1), 4, (14, 10, 10), (False, False, False)],
@@ -168,17 +153,9 @@ def test_conv2d():
         )
 
 
-def test_batchnorm():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    np.random.seed(0)
-
-    dtype = "float32"
-
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def _test_batchnorm(device, dtype):
     in_shape = (1, 8, 64, 64)
     channels = 8
 
@@ -211,14 +188,9 @@ def test_batchnorm():
     )
 
 
-def test_concat():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    dtype = "float16"
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def test_concat(device, dtype):
     in_shape_1 = (1, 16, 16, 16)
     in_shape_2 = (1, 16, 16, 16)
     a = relay.var("input_1", shape=in_shape_1, dtype=dtype)
@@ -241,14 +213,9 @@ def test_concat():
     )
 
 
-def test_avgpool():
-    Device.load("test_config.json")
-
-    if skip_runtime_test():
-        return
-
-    device = Device()
-    dtype = "float16"
+@pytest.mark.parametrize("dtype", ["float16"])
+@tvm.testing.requires_openclml
+def test_avgpool(device, dtype):
     trials = [
         # input size         pool_size stride  paading
         [(1, 64, 147, 147), (3, 3), (2, 2), (0, 0, 0, 0), "max"],
@@ -288,10 +255,3 @@ def test_avgpool():
         tvm.testing.assert_allclose(
             clml_out[0].asnumpy(), opencl_out[0].asnumpy(), rtol=1e-3, atol=1e-3
         )
-
-
-if __name__ == "__main__":
-    test_conv2d()
-    # test_batchnorm()
-    test_avgpool()
-    test_concat()

tests/scripts/task_python_adreno.sh

@@ -38,14 +38,14 @@ sleep 5   # Wait for tracker to bind
 export ANDROID_SERIAL=$1
 
 adb shell "mkdir -p /data/local/tmp/tvm_ci"
-adb push build-adreno-target/tvm_rpc /data/local/tmp/tvm_ci
+adb push build-adreno-target/tvm_rpc /data/local/tmp/tvm_ci/tvm_rpc_ci
 adb push build-adreno-target/libtvm_runtime.so /data/local/tmp/tvm_ci
 
 adb reverse tcp:${TVM_TRACKER_PORT} tcp:${TVM_TRACKER_PORT}
 adb forward tcp:5000 tcp:5000
 adb forward tcp:5001 tcp:5001
 adb forward tcp:5002 tcp:5002
-env adb shell "cd /data/local/tmp/tvm_ci; killall -9 tvm_rpc; sleep 2; LD_LIBRARY_PATH=/data/local/tmp/tvm_ci/ ./tvm_rpc server --host=0.0.0.0 --port=5000 --port-end=5010 --tracker=127.0.0.1:${TVM_TRACKER_PORT} --key=android" &
+env adb shell "cd /data/local/tmp/tvm_ci; killall -9 tvm_rpc_ci; sleep 2; LD_LIBRARY_PATH=/data/local/tmp/tvm_ci/ ./tvm_rpc_ci server --host=0.0.0.0 --port=5000 --port-end=5010 --tracker=127.0.0.1:${TVM_TRACKER_PORT} --key=android" &
 DEVICE_PID=$!
 sleep 5 # Wait for the device connections
 
@@ -58,5 +58,8 @@ make cython3
 # OpenCL texture test on Adreno
 run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-opencl-texture tests/python/relay/opencl_texture
 
+# Adreno CLML test
+run_pytest ctypes ${TVM_INTEGRATION_TESTSUITE_NAME}-openclml tests/python/contrib/test_clml
+
 kill ${TRACKER_PID}
 kill ${DEVICE_PID}