From fd99b58c230c7d8a8fcad1a53aedc107359ac32f Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Sun, 3 Mar 2024 07:34:01 -0800 Subject: [PATCH] Run get_concurrent_jobs.py as a gclient hook --- .gitignore | 5 -- DEPS | 36 --------- build/concurrent_jobs.gni | 12 --- build/dart/BUILD.gn | 3 +- build/get_concurrent_jobs.py | 130 ------------------------------ build/git_revision.py | 8 -- ci/licenses_golden/excluded_files | 3 - ci/licenses_golden/tool_signature | 2 +- impeller/tools/BUILD.gn | 14 +--- tools/gn | 86 ++++++++++++++++++-- tools/licenses/lib/paths.dart | 3 - 11 files changed, 82 insertions(+), 220 deletions(-) delete mode 100644 build/concurrent_jobs.gni delete mode 100755 build/get_concurrent_jobs.py diff --git a/.gitignore b/.gitignore index e0a6d2d6941b9..aa52914644cac 100644 --- a/.gitignore +++ b/.gitignore @@ -142,8 +142,3 @@ app.*.symbols # The gn-sdk from Chromium and managed by DEPS and gclient. /tools/fuchsia/gn-sdk - -# Git revision files created by gclient runhooks -engine.revision -dart.revision -skia.revision diff --git a/DEPS b/DEPS index e9e411f73c58c..fce02b38a2c29 100644 --- a/DEPS +++ b/DEPS @@ -1236,41 +1236,5 @@ hooks = [ '--as-gclient-hook', Var('mac_sdk_min') ] - }, - { - 'name': 'Get Engine git revision', - 'pattern': '.', - 'action': [ - 'python3', - 'src/flutter/build/git_revision.py', - '--repository', - 'src/flutter', - '--output', - 'src/flutter/engine.revision', - ] - }, - { - 'name': 'Get Skia git revision', - 'pattern': '.', - 'action': [ - 'python3', - 'src/flutter/build/git_revision.py', - '--repository', - 'src/flutter/third_party/skia', - '--output', - 'src/flutter/skia.revision', - ] - }, - { - 'name': 'Get Dart git revision', - 'pattern': '.', - 'action': [ - 'python3', - 'src/flutter/build/git_revision.py', - '--repository', - 'src/third_party/dart', - '--output', - 'src/flutter/dart.revision', - ] } ] diff --git a/build/concurrent_jobs.gni b/build/concurrent_jobs.gni deleted file mode 100644 index a930b25568276..0000000000000 --- a/build/concurrent_jobs.gni +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -_script = "//flutter/build/get_concurrent_jobs.py" -_args = [ - "--reserve-memory=1GB", - "--memory-per-job", - "dart=1GB", -] - -concurrent_jobs = exec_script(_script, _args, "json", [ _script ]) diff --git a/build/dart/BUILD.gn b/build/dart/BUILD.gn index a8692ab08b15f..4c4d57d06965d 100644 --- a/build/dart/BUILD.gn +++ b/build/dart/BUILD.gn @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//flutter/build/concurrent_jobs.gni") import("//flutter/build/dart/dart.gni") import("//flutter/common/config.gni") @@ -12,7 +11,7 @@ declare_args() { # Maximum number of Dart processes to run in parallel. # # To avoid out-of-memory errors we explicitly reduce the number of jobs. - concurrent_dart_jobs = concurrent_jobs.dart + concurrent_dart_jobs = 1 } pool("dart_pool") { diff --git a/build/get_concurrent_jobs.py b/build/get_concurrent_jobs.py deleted file mode 100755 index 3559073c66dce..0000000000000 --- a/build/get_concurrent_jobs.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Fuchsia Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script computes the number of concurrent jobs that can run in the -# build as a function of the machine. It accepts a set of key value pairs -# given by repeated --memory-per-job arguments. For example: -# -# $ get_concurrent_jobs.py --memory-per-job dart=1GB -# -# The result is a json map printed to stdout that gives the number of -# concurrent jobs allowed of each kind. For example: -# -# {"dart": 8} -# -# Some memory can be held out of the calculation with the --reserve-memory flag. - -import argparse -import ctypes -import json -import multiprocessing -import os -import re -import subprocess -import sys - -UNITS = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40} - - -# pylint: disable=line-too-long -# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex -# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex -# pylint: enable=line-too-long -class MEMORYSTATUSEX(ctypes.Structure): - _fields_ = [ - ('dwLength', ctypes.c_ulong), - ('dwMemoryLoad', ctypes.c_ulong), - ('ullTotalPhys', ctypes.c_ulonglong), - ('ullAvailPhys', ctypes.c_ulonglong), - ('ullTotalPageFile', ctypes.c_ulonglong), - ('ullAvailPageFile', ctypes.c_ulonglong), - ('ullTotalVirtual', ctypes.c_ulonglong), - ('ullAvailVirtual', ctypes.c_ulonglong), - ('sullAvailExtendedVirtual', ctypes.c_ulonglong), - ] - - -def get_total_memory(): - if sys.platform in ('win32', 'cygwin'): - stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) - success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) - return stat.ullTotalPhys if success else 0 - if sys.platform.startswith('linux'): - if os.path.exists('/proc/meminfo'): - with open('/proc/meminfo') as meminfo: - memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') - for line in meminfo: - match = memtotal_re.match(line) - if match: - return float(match.group(1)) * 2**10 - if sys.platform == 'darwin': - try: - return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) - except: # pylint: disable=bare-except - return 0 - return 0 - - -def parse_size(string): - i = next(i for (i, c) in enumerate(string) if not c.isdigit()) - number = string[:i].strip() - unit = string[i:].strip() - return int(float(number) * UNITS[unit]) - - -class ParseSizeAction(argparse.Action): - - def __call__(self, parser, args, values, option_string=None): - sizes = getattr(args, self.dest, []) - for value in values: - (k, val) = value.split('=', 1) - sizes.append((k, parse_size(val))) - setattr(args, self.dest, sizes) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--memory-per-job', - action=ParseSizeAction, - default=[], - nargs='*', - help='Key value pairings (dart=1GB) giving an estimate of the amount of ' - 'memory needed for the class of job.' - ) - parser.add_argument( - '--reserve-memory', - type=parse_size, - default=0, - help='The amount of memory to be held out of the amount for jobs to use.' - ) - args = parser.parse_args() - - total_memory = get_total_memory() - - # Ensure the total memory used in the calculation below is at least 0 - mem_total_bytes = max(0, total_memory - args.reserve_memory) - - # Ensure the number of cpus used in the calculation below is at least 1 - try: - cpu_cap = multiprocessing.cpu_count() - except: # pylint: disable=bare-except - cpu_cap = 1 - - concurrent_jobs = {} - for job, memory_per_job in args.memory_per_job: - # Calculate the number of jobs that will fit in memory. Ensure the - # value is at least 1. - num_concurrent_jobs = int(max(1, mem_total_bytes / memory_per_job)) - # Cap the number of jobs by the number of cpus available. - concurrent_jobs[job] = min(num_concurrent_jobs, cpu_cap) - - print(json.dumps(concurrent_jobs)) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/build/git_revision.py b/build/git_revision.py index 73fdca6b12e0a..8f7759e51f0fa 100755 --- a/build/git_revision.py +++ b/build/git_revision.py @@ -42,18 +42,10 @@ def main(): parser.add_argument( '--repository', action='store', help='Path to the Git repository.', required=True ) - parser.add_argument( - '--output', action='store', help='Write revision to a file at this path instead of printing.' - ) args = parser.parse_args() repository = os.path.abspath(args.repository) version = get_repository_version(repository) - - if args.output: - with open(args.output, 'w') as file: - file.write(version.strip()) - print(version.strip()) return 0 diff --git a/ci/licenses_golden/excluded_files b/ci/licenses_golden/excluded_files index fd13a3670b34b..1b6ddd746d4fb 100644 --- a/ci/licenses_golden/excluded_files +++ b/ci/licenses_golden/excluded_files @@ -28,7 +28,6 @@ ../../../flutter/build ../../../flutter/ci ../../../flutter/common/README.md -../../../flutter/dart.revision ../../../flutter/display_list/benchmarking/dl_complexity_unittests.cc ../../../flutter/display_list/display_list_unittests.cc ../../../flutter/display_list/dl_color_unittests.cc @@ -46,7 +45,6 @@ ../../../flutter/display_list/testing ../../../flutter/display_list/utils/dl_matrix_clip_tracker_unittests.cc ../../../flutter/docs -../../../flutter/engine.revision ../../../flutter/examples ../../../flutter/flow/README.md ../../../flutter/flow/diff_context_unittests.cc @@ -404,7 +402,6 @@ ../../../flutter/shell/vmservice/.dart_tool ../../../flutter/shell/vmservice/pubspec.lock ../../../flutter/shell/vmservice/pubspec.yaml -../../../flutter/skia.revision ../../../flutter/sky/packages/sky_engine/.gitignore ../../../flutter/sky/packages/sky_engine/LICENSE ../../../flutter/sky/packages/sky_engine/README.md diff --git a/ci/licenses_golden/tool_signature b/ci/licenses_golden/tool_signature index eaef080de940d..2178fa581da37 100644 --- a/ci/licenses_golden/tool_signature +++ b/ci/licenses_golden/tool_signature @@ -1,2 +1,2 @@ -Signature: 9c3527789c4195b860c094208c53f101 +Signature: 04fdc5b7c8d5b1690149c9aa20f4174d diff --git a/impeller/tools/BUILD.gn b/impeller/tools/BUILD.gn index a8bf1e0068548..ae1bf978241ab 100644 --- a/impeller/tools/BUILD.gn +++ b/impeller/tools/BUILD.gn @@ -11,19 +11,7 @@ declare_args() { # Maximum number of malioc processes to run in parallel. # # To avoid out-of-memory errors we explicitly reduce the number of jobs. - impeller_concurrent_malioc_jobs = -1 -} - -if (impeller_concurrent_malioc_jobs == -1) { - _script = "//flutter/build/get_concurrent_jobs.py" - _args = [ - "--reserve-memory=1GB", - "--memory-per-job", - "malioc=100MB", - ] - _concurrent_jobs = exec_script(_script, _args, "json", [ _script ]) - impeller_concurrent_malioc_jobs = _concurrent_jobs.malioc - assert(impeller_concurrent_malioc_jobs > 0) + impeller_concurrent_malioc_jobs = 1 } pool("malioc_pool") { diff --git a/tools/gn b/tools/gn index bddc58333cd43..375af5d707939 100755 --- a/tools/gn +++ b/tools/gn @@ -8,10 +8,13 @@ from __future__ import division from __future__ import print_function import argparse -import subprocess -import sys +import ctypes +import multiprocessing import os import platform +import re +import subprocess +import sys SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -378,6 +381,73 @@ def setup_git_versions(): return revision_args +# pylint: disable=line-too-long +# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex +# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex +# pylint: enable=line-too-long +class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ('dwLength', ctypes.c_ulong), + ('dwMemoryLoad', ctypes.c_ulong), + ('ullTotalPhys', ctypes.c_ulonglong), + ('ullAvailPhys', ctypes.c_ulonglong), + ('ullTotalPageFile', ctypes.c_ulonglong), + ('ullAvailPageFile', ctypes.c_ulonglong), + ('ullTotalVirtual', ctypes.c_ulonglong), + ('ullAvailVirtual', ctypes.c_ulonglong), + ('sullAvailExtendedVirtual', ctypes.c_ulonglong), + ] + + +def get_total_memory(): + if sys.platform in ('win32', 'cygwin'): + stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) + success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) + return stat.ullTotalPhys if success else 0 + if sys.platform.startswith('linux'): + if os.path.exists('/proc/meminfo'): + with open('/proc/meminfo') as meminfo: + memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') + for line in meminfo: + match = memtotal_re.match(line) + if match: + return float(match.group(1)) * 2**10 + if sys.platform == 'darwin': + try: + return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) + except: # pylint: disable=bare-except + return 0 + return 0 + + +def parse_size(string): + units = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40} + i = next(i for (i, c) in enumerate(string) if not c.isdigit()) + number = string[:i].strip() + unit = string[i:].strip() + return int(float(number) * units[unit]) + + +def get_concurrent_jobs(reserve_memory, memory_per_job): + # reserve_memory = 2**30 + total_memory = get_total_memory() + # Ensure the total memory used in the calculation below is at least 0 + mem_total_bytes = max(0, total_memory - parse_size(reserve_memory)) + # Ensure the number of cpus used in the calculation below is at least 1 + try: + cpu_cap = multiprocessing.cpu_count() + except: # pylint: disable=bare-except + cpu_cap = 1 + + # Calculate the number of jobs that will fit in memory. Ensure the + # value is at least 1. + num_concurrent_jobs = int(max(1, mem_total_bytes / parse_size(memory_per_job))) + # Cap the number of jobs by the number of cpus available. + concurrent_jobs = min(num_concurrent_jobs, cpu_cap) + + return concurrent_jobs + + def to_gn_args(args): if args.simulator: if args.target_os != 'ios': @@ -529,6 +599,8 @@ def to_gn_args(args): else: gn_args['dart_runtime_mode'] = runtime_mode + gn_args['concurrent_dart_jobs'] = get_concurrent_jobs('1GB', '1GB') + # Desktop embeddings can have more dependencies than the engine library, # which can be problematic in some build environments (e.g., building on # Linux will bring in pkg-config dependencies at generation time). These @@ -712,12 +784,12 @@ def to_gn_args(args): if args.prebuilt_impellerc is not None: gn_args['impeller_use_prebuilt_impellerc'] = args.prebuilt_impellerc - if args.malioc_path is not None: - gn_args['impeller_malioc_path'] = args.malioc_path - else: + malioc_path = args.malioc_path + if not malioc_path: malioc_path = os.environ.get('MALIOC_PATH') - if malioc_path: - gn_args['impeller_malioc_path'] = malioc_path + if malioc_path: + gn_args['impeller_malioc_path'] = malioc_path + gn_args['impeller_concurrent_malioc_jobs'] = get_concurrent_jobs('1GB', '100MB') if args.use_glfw_swiftshader: if get_host_os() == 'mac': diff --git a/tools/licenses/lib/paths.dart b/tools/licenses/lib/paths.dart index b0c117154f03e..6fb05be72f844 100644 --- a/tools/licenses/lib/paths.dart +++ b/tools/licenses/lib/paths.dart @@ -22,15 +22,12 @@ final Set skippedPaths = { r'buildtools', // only used by build r'flutter/build', r'flutter/ci', - r'flutter/dart.revision', // only used by build r'flutter/docs', - r'flutter/engine.revision', // only used by build r'flutter/flutter_frontend_server', r'flutter/impeller/docs', r'flutter/lib/web_ui/build', // this is compiler-generated output r'flutter/lib/web_ui/dev', // these are build tools; they do not end up in Engine artifacts r'flutter/prebuilts', - r'flutter/skia.revision', // only used by build r'flutter/sky/packages/sky_engine/LICENSE', r'flutter/third_party/benchmark', // only used by tests r'flutter/third_party/boringssl/src/crypto/err/err_data_generate.go',