From e01b21e3b06139411fc3cb4328d564395512b8f0 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 21:53:34 +0200 Subject: [PATCH 01/11] Add `--hard-fail` list argument to benchmarks for CI Will cause CI to fail on a benchmark failure for given indices. --- utils/benchmarks.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index 69d653a20916..68000f028752 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -54,6 +54,7 @@ def run( half=False, # use FP16 half-precision inference test=False, # test exports only pt_only=False, # test PyTorch only + hard_fail=[], # throw error for formats: --hard-fail 0, or 0 2 3 ): y, t = [], time.time() device = select_device(device) @@ -78,6 +79,8 @@ def run( y.append([name, round(file_size(w), 1), round(metrics[3], 4), round(speeds[1], 2)]) # MB, mAP, t_inference except Exception as e: LOGGER.warning(f'WARNING: Benchmark failure for {name}: {e}') + if i in hard_fail: + Exception(e) y.append([name, None, None, None]) # mAP, t_inference if pt_only and i == 0: break # break after PyTorch @@ -102,6 +105,7 @@ def test( half=False, # use FP16 half-precision inference test=False, # test exports only pt_only=False, # test PyTorch only + hard_fail=[], # throw error for formats: --hard-fail 0, or 0 2 3 ): y, t = [], time.time() device = select_device(device) @@ -134,6 +138,7 @@ def parse_opt(): parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') parser.add_argument('--test', action='store_true', help='test exports only') parser.add_argument('--pt-only', action='store_true', help='test PyTorch only') + parser.add_argument('--hard-fail', nargs='+', default=[], help='throw error for formats: --hard-fail 0, or 0 2 3') opt = parser.parse_args() opt.data = check_yaml(opt.data) # check YAML print_args(vars(opt)) From aa068c5f5424b3f1e7d69def9350508da034854f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 21:55:02 +0200 Subject: [PATCH 02/11] Update ci-testing.yml --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 4083ac354c46..b59b6469ecbd 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -39,7 +39,7 @@ jobs: pip list - name: Run benchmarks run: | - python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 + python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 --hard-fail 0 1 2 3 6 7 8 Tests: timeout-minutes: 60 From 43d4976a3fd9c2e3e738c38289e483644cdf1021 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 21:55:51 +0200 Subject: [PATCH 03/11] Attempt Failure (CI should fail) --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index b59b6469ecbd..727cd9881445 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -39,7 +39,7 @@ jobs: pip list - name: Run benchmarks run: | - python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 --hard-fail 0 1 2 3 6 7 8 + python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 --hard-fail 0 1 2 3 4 6 7 8 Tests: timeout-minutes: 60 From 4e547ef5e20d8cf7d60cd18b740a5f19dcc5090f Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 22:46:19 +0200 Subject: [PATCH 04/11] Update benchmarks.py --- utils/benchmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index 68000f028752..9db2812724d6 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -80,7 +80,7 @@ def run( except Exception as e: LOGGER.warning(f'WARNING: Benchmark failure for {name}: {e}') if i in hard_fail: - Exception(e) + raise Exception(e) y.append([name, None, None, None]) # mAP, t_inference if pt_only and i == 0: break # break after PyTorch From a137dd3844c17b694f74f79227023222957983a2 Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 22:58:45 +0200 Subject: [PATCH 05/11] Update export.py --- export.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/export.py b/export.py index 9daf39f871c2..275c866ff856 100644 --- a/export.py +++ b/export.py @@ -75,18 +75,18 @@ def export_formats(): # YOLOv5 export formats x = [ - ['PyTorch', '-', '.pt', True], - ['TorchScript', 'torchscript', '.torchscript', True], - ['ONNX', 'onnx', '.onnx', True], - ['OpenVINO', 'openvino', '_openvino_model', False], - ['TensorRT', 'engine', '.engine', True], - ['CoreML', 'coreml', '.mlmodel', False], - ['TensorFlow SavedModel', 'saved_model', '_saved_model', True], - ['TensorFlow GraphDef', 'pb', '.pb', True], - ['TensorFlow Lite', 'tflite', '.tflite', False], - ['TensorFlow Edge TPU', 'edgetpu', '_edgetpu.tflite', False], - ['TensorFlow.js', 'tfjs', '_web_model', False],] - return pd.DataFrame(x, columns=['Format', 'Argument', 'Suffix', 'GPU']) + ['PyTorch', '-', '.pt', True, True], + ['TorchScript', 'torchscript', '.torchscript', True, True], + ['ONNX', 'onnx', '.onnx', True, True], + ['OpenVINO', 'openvino', '_openvino_model', True, False], + ['TensorRT', 'engine', '.engine', False, True], + ['CoreML', 'coreml', '.mlmodel', True, False], + ['TensorFlow SavedModel', 'saved_model', '_saved_model', True, True], + ['TensorFlow GraphDef', 'pb', '.pb', True, True], + ['TensorFlow Lite', 'tflite', '.tflite', True, False], + ['TensorFlow Edge TPU', 'edgetpu', '_edgetpu.tflite', False, False], + ['TensorFlow.js', 'tfjs', '_web_model', False, False],] + return pd.DataFrame(x, columns=['Format', 'Argument', 'Suffix', 'CPU', 'GPU']) def export_torchscript(model, im, file, optimize, prefix=colorstr('TorchScript:')): @@ -436,9 +436,9 @@ def export_tfjs(file, prefix=colorstr('TensorFlow.js:')): r'"Identity.?.?": {"name": "Identity.?.?"}, ' r'"Identity.?.?": {"name": "Identity.?.?"}, ' r'"Identity.?.?": {"name": "Identity.?.?"}}}', r'{"outputs": {"Identity": {"name": "Identity"}, ' - r'"Identity_1": {"name": "Identity_1"}, ' - r'"Identity_2": {"name": "Identity_2"}, ' - r'"Identity_3": {"name": "Identity_3"}}}', json) + r'"Identity_1": {"name": "Identity_1"}, ' + r'"Identity_2": {"name": "Identity_2"}, ' + r'"Identity_3": {"name": "Identity_3"}}}', json) j.write(subst) LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') From f37818aa53a3fcca1ecf3eb6b2131dcc57768dce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 20:59:05 +0000 Subject: [PATCH 06/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- export.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/export.py b/export.py index 275c866ff856..1d8f07fc9e2f 100644 --- a/export.py +++ b/export.py @@ -436,9 +436,9 @@ def export_tfjs(file, prefix=colorstr('TensorFlow.js:')): r'"Identity.?.?": {"name": "Identity.?.?"}, ' r'"Identity.?.?": {"name": "Identity.?.?"}, ' r'"Identity.?.?": {"name": "Identity.?.?"}}}', r'{"outputs": {"Identity": {"name": "Identity"}, ' - r'"Identity_1": {"name": "Identity_1"}, ' - r'"Identity_2": {"name": "Identity_2"}, ' - r'"Identity_3": {"name": "Identity_3"}}}', json) + r'"Identity_1": {"name": "Identity_1"}, ' + r'"Identity_2": {"name": "Identity_2"}, ' + r'"Identity_3": {"name": "Identity_3"}}}', json) j.write(subst) LOGGER.info(f'{prefix} export success, saved as {f} ({file_size(f):.1f} MB)') From 01bf659d454982d9ae839ee4563b25b70646649c Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 23:13:01 +0200 Subject: [PATCH 07/11] Update benchmarks.py --- utils/benchmarks.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index 9db2812724d6..c3015eba4e8e 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -54,16 +54,18 @@ def run( half=False, # use FP16 half-precision inference test=False, # test exports only pt_only=False, # test PyTorch only - hard_fail=[], # throw error for formats: --hard-fail 0, or 0 2 3 + hard_fail=False, # throw error on benchmark failure ): y, t = [], time.time() device = select_device(device) - for i, (name, f, suffix, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable) + for i, ( + name, f, suffix, cpu, + gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable) try: assert i != 9, 'Edge TPU not supported' assert i != 10, 'TF.js not supported' - if device.type != 'cpu': - assert gpu, f'{name} inference not supported on GPU' + assert cpu or 'cpu' not in device.type, f'{name} inference not supported on CPU' + assert gpu or 'gpu' not in device.type, f'{name} inference not supported on GPU' # Export if f == '-': @@ -78,9 +80,9 @@ def run( speeds = result[2] # times (preprocess, inference, postprocess) y.append([name, round(file_size(w), 1), round(metrics[3], 4), round(speeds[1], 2)]) # MB, mAP, t_inference except Exception as e: + if hard_fail: + assert type(e) is AssertionError, f'Benchmark --hard-fail for {name}: {e}' LOGGER.warning(f'WARNING: Benchmark failure for {name}: {e}') - if i in hard_fail: - raise Exception(e) y.append([name, None, None, None]) # mAP, t_inference if pt_only and i == 0: break # break after PyTorch @@ -105,7 +107,7 @@ def test( half=False, # use FP16 half-precision inference test=False, # test exports only pt_only=False, # test PyTorch only - hard_fail=[], # throw error for formats: --hard-fail 0, or 0 2 3 + hard_fail=False, # throw error on benchmark failure ): y, t = [], time.time() device = select_device(device) @@ -138,7 +140,7 @@ def parse_opt(): parser.add_argument('--half', action='store_true', help='use FP16 half-precision inference') parser.add_argument('--test', action='store_true', help='test exports only') parser.add_argument('--pt-only', action='store_true', help='test PyTorch only') - parser.add_argument('--hard-fail', nargs='+', default=[], help='throw error for formats: --hard-fail 0, or 0 2 3') + parser.add_argument('--hard-fail', action='store_true', help='throw error on benchmark failure') opt = parser.parse_args() opt.data = check_yaml(opt.data) # check YAML print_args(vars(opt)) From 64e37919055f2829d2821d4d3b7c9ada6123cf0e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 21:13:21 +0000 Subject: [PATCH 08/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- utils/benchmarks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index c3015eba4e8e..c86989d0181c 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -58,8 +58,7 @@ def run( ): y, t = [], time.time() device = select_device(device) - for i, ( - name, f, suffix, cpu, + for i, (name, f, suffix, cpu, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable) try: assert i != 9, 'Edge TPU not supported' From 1591ab1ddac95e6925ee34e9da6ce6a7fda3e28d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 23:13:39 +0200 Subject: [PATCH 09/11] Update ci-testing.yml --- .github/workflows/ci-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 727cd9881445..f3e36675f49d 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -39,7 +39,7 @@ jobs: pip list - name: Run benchmarks run: | - python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 --hard-fail 0 1 2 3 4 6 7 8 + python utils/benchmarks.py --weights ${{ matrix.model }}.pt --img 320 --hard-fail Tests: timeout-minutes: 60 From 2c4d7271378775951190e7454f548fe4d53349fc Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 23:23:33 +0200 Subject: [PATCH 10/11] Update benchmarks.py --- utils/benchmarks.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index c86989d0181c..54cbb87a8662 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -26,6 +26,7 @@ """ import argparse +import platform import sys import time from pathlib import Path @@ -58,12 +59,13 @@ def run( ): y, t = [], time.time() device = select_device(device) - for i, (name, f, suffix, cpu, - gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, gpu-capable) + for i, (name, f, suffix, cpu, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, CPU, GPU) try: assert i != 9, 'Edge TPU not supported' assert i != 10, 'TF.js not supported' - assert cpu or 'cpu' not in device.type, f'{name} inference not supported on CPU' + if cpu: + assert 'cpu' in device.type, f'{name} inference not supported on CPU' + assert i != 5 or platform.system() == 'Darwin', f'{name} inference only supported on macOS>=10.13' assert gpu or 'gpu' not in device.type, f'{name} inference not supported on GPU' # Export From c64e5bdd5fc5529419b2d1c64bd46bcb2c1dd39d Mon Sep 17 00:00:00 2001 From: Glenn Jocher Date: Thu, 7 Jul 2022 23:33:07 +0200 Subject: [PATCH 11/11] Update benchmarks.py --- utils/benchmarks.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/benchmarks.py b/utils/benchmarks.py index 54cbb87a8662..03bab9b6ded2 100644 --- a/utils/benchmarks.py +++ b/utils/benchmarks.py @@ -61,12 +61,12 @@ def run( device = select_device(device) for i, (name, f, suffix, cpu, gpu) in export.export_formats().iterrows(): # index, (name, file, suffix, CPU, GPU) try: - assert i != 9, 'Edge TPU not supported' - assert i != 10, 'TF.js not supported' - if cpu: - assert 'cpu' in device.type, f'{name} inference not supported on CPU' - assert i != 5 or platform.system() == 'Darwin', f'{name} inference only supported on macOS>=10.13' - assert gpu or 'gpu' not in device.type, f'{name} inference not supported on GPU' + assert i not in (9, 10), f'{name} inference not supported' # Edge TPU and TF.js are unsupported + assert i != 5 or platform.system() == 'Darwin', f'{name} inference only supported on macOS>=10.13' + if 'cpu' in device.type: + assert cpu, f'{name} inference not supported on CPU' + if 'cuda' in device.type: + assert gpu, f'{name} inference not supported on GPU' # Export if f == '-':