From 139f2cae038b754051efa4d29d0504cc4f385ead Mon Sep 17 00:00:00 2001 From: "[[ -z $EMAIL ]] && read -e -p \"Enter your email (for git configuration): \" EMAIL" Date: Thu, 6 Feb 2025 12:22:44 -0500 Subject: [PATCH 1/2] Brr --- utils/check_modular_conversion.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/check_modular_conversion.py b/utils/check_modular_conversion.py index 5946d6ef1687..c4d7e6c72c16 100644 --- a/utils/check_modular_conversion.py +++ b/utils/check_modular_conversion.py @@ -2,7 +2,9 @@ import difflib import glob import logging +import multiprocessing from io import StringIO +from multiprocessing import Pool from create_dependency_mapping import find_priority_list @@ -52,7 +54,8 @@ def process_file(modular_file_path, generated_modeling_content, file_type="model return 0 -def compare_files(modular_file_path, fix_and_overwrite=False): +def compare_files(args): + modular_file_path, fix_and_overwrite = args # Generate the expected modeling content generated_modeling_content = convert_modular_file(modular_file_path) diff = 0 @@ -72,9 +75,11 @@ def compare_files(modular_file_path, fix_and_overwrite=False): args = parser.parse_args() if args.files == ["all"]: args.files = glob.glob("src/transformers/models/**/modular_*.py", recursive=True) - non_matching_files = 0 - for modular_file_path in find_priority_list(args.files): - non_matching_files += compare_files(modular_file_path, args.fix_and_overwrite) + + num_cores = max(1, multiprocessing.cpu_count() - 1) # Use all cores except one to avoid overloading the system + with Pool(num_cores) as pool: + files_with_args = [(f, args.fix_and_overwrite) for f in find_priority_list(args.files)] + non_matching_files = sum(pool.map(compare_files, files_with_args)) if non_matching_files and not args.fix_and_overwrite: - raise ValueError("Some diff and their modeling code did not match.") + raise ValueError("Some diff and their modeling code did not match.") \ No newline at end of file From c631f36ccae1d6233e8ad0fbfc7bc046c7a15f31 Mon Sep 17 00:00:00 2001 From: "[[ -z $EMAIL ]] && read -e -p \"Enter your email (for git configuration): \" EMAIL" Date: Thu, 6 Feb 2025 12:26:38 -0500 Subject: [PATCH 2/2] quality --- utils/check_modular_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check_modular_conversion.py b/utils/check_modular_conversion.py index c4d7e6c72c16..8d71baa82634 100644 --- a/utils/check_modular_conversion.py +++ b/utils/check_modular_conversion.py @@ -82,4 +82,4 @@ def compare_files(args): non_matching_files = sum(pool.map(compare_files, files_with_args)) if non_matching_files and not args.fix_and_overwrite: - raise ValueError("Some diff and their modeling code did not match.") \ No newline at end of file + raise ValueError("Some diff and their modeling code did not match.")