Run get_concurrent_jobs.py as a gclient hook

.gitignore

@@ -142,8 +142,3 @@ app.*.symbols
 
 # The gn-sdk from Chromium and managed by DEPS and gclient.
 /tools/fuchsia/gn-sdk
-
-# Git revision files created by gclient runhooks
-engine.revision
-dart.revision
-skia.revision

DEPS

@@ -1236,41 +1236,5 @@ hooks = [
       '--as-gclient-hook',
       Var('mac_sdk_min')
     ]
-  },
-  {
-    'name': 'Get Engine git revision',
-    'pattern': '.',
-    'action': [
-      'python3',
-      'src/flutter/build/git_revision.py',
-      '--repository',
-      'src/flutter',
-      '--output',
-      'src/flutter/engine.revision',
-    ]
-  },
-  {
-    'name': 'Get Skia git revision',
-    'pattern': '.',
-    'action': [
-      'python3',
-      'src/flutter/build/git_revision.py',
-      '--repository',
-      'src/flutter/third_party/skia',
-      '--output',
-      'src/flutter/skia.revision',
-    ]
-  },
-  {
-    'name': 'Get Dart git revision',
-    'pattern': '.',
-    'action': [
-      'python3',
-      'src/flutter/build/git_revision.py',
-      '--repository',
-      'src/third_party/dart',
-      '--output',
-      'src/flutter/dart.revision',
-    ]
   }
 ]

build/dart/BUILD.gn

@@ -2,7 +2,6 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import("//flutter/build/concurrent_jobs.gni")
 import("//flutter/build/dart/dart.gni")
 import("//flutter/common/config.gni")
 
@@ -12,7 +11,7 @@ declare_args() {
   # Maximum number of Dart processes to run in parallel.
   #
   # To avoid out-of-memory errors we explicitly reduce the number of jobs.
-  concurrent_dart_jobs = concurrent_jobs.dart
+  concurrent_dart_jobs = 1
 }
 
 pool("dart_pool") {

build/git_revision.py

@@ -42,18 +42,10 @@ def main():
   parser.add_argument(
       '--repository', action='store', help='Path to the Git repository.', required=True
   )
-  parser.add_argument(
-      '--output', action='store', help='Write revision to a file at this path instead of printing.'
-  )
 
   args = parser.parse_args()
   repository = os.path.abspath(args.repository)
   version = get_repository_version(repository)
-
-  if args.output:
-    with open(args.output, 'w') as file:
-      file.write(version.strip())
-
   print(version.strip())
 
   return 0

ci/licenses_golden/excluded_files

@@ -28,7 +28,6 @@
 ../../../flutter/build
 ../../../flutter/ci
 ../../../flutter/common/README.md
-../../../flutter/dart.revision
 ../../../flutter/display_list/benchmarking/dl_complexity_unittests.cc
 ../../../flutter/display_list/display_list_unittests.cc
 ../../../flutter/display_list/dl_color_unittests.cc
@@ -46,7 +45,6 @@
 ../../../flutter/display_list/testing
 ../../../flutter/display_list/utils/dl_matrix_clip_tracker_unittests.cc
 ../../../flutter/docs
-../../../flutter/engine.revision
 ../../../flutter/examples
 ../../../flutter/flow/README.md
 ../../../flutter/flow/diff_context_unittests.cc
@@ -404,7 +402,6 @@
 ../../../flutter/shell/vmservice/.dart_tool
 ../../../flutter/shell/vmservice/pubspec.lock
 ../../../flutter/shell/vmservice/pubspec.yaml
-../../../flutter/skia.revision
 ../../../flutter/sky/packages/sky_engine/.gitignore
 ../../../flutter/sky/packages/sky_engine/LICENSE
 ../../../flutter/sky/packages/sky_engine/README.md

ci/licenses_golden/tool_signature

@@ -1,2 +1,2 @@
-Signature: 9c3527789c4195b860c094208c53f101
+Signature: 04fdc5b7c8d5b1690149c9aa20f4174d
 

impeller/tools/BUILD.gn

@@ -11,19 +11,7 @@ declare_args() {
   # Maximum number of malioc processes to run in parallel.
   #
   # To avoid out-of-memory errors we explicitly reduce the number of jobs.
-  impeller_concurrent_malioc_jobs = -1
-}
-
-if (impeller_concurrent_malioc_jobs == -1) {
-  _script = "//flutter/build/get_concurrent_jobs.py"
-  _args = [
-    "--reserve-memory=1GB",
-    "--memory-per-job",
-    "malioc=100MB",
-  ]
-  _concurrent_jobs = exec_script(_script, _args, "json", [ _script ])
-  impeller_concurrent_malioc_jobs = _concurrent_jobs.malioc
-  assert(impeller_concurrent_malioc_jobs > 0)
+  impeller_concurrent_malioc_jobs = 1
 }
 
 pool("malioc_pool") {

tools/gn

@@ -8,10 +8,13 @@ from __future__ import division
 from __future__ import print_function
 
 import argparse
-import subprocess
-import sys
+import ctypes
+import multiprocessing
 import os
 import platform
+import re
+import subprocess
+import sys
 
 SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -378,6 +381,73 @@ def setup_git_versions():
   return revision_args
 
 
+# pylint: disable=line-too-long
+# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
+# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
+# pylint: enable=line-too-long
+class MEMORYSTATUSEX(ctypes.Structure):
+  _fields_ = [
+      ('dwLength', ctypes.c_ulong),
+      ('dwMemoryLoad', ctypes.c_ulong),
+      ('ullTotalPhys', ctypes.c_ulonglong),
+      ('ullAvailPhys', ctypes.c_ulonglong),
+      ('ullTotalPageFile', ctypes.c_ulonglong),
+      ('ullAvailPageFile', ctypes.c_ulonglong),
+      ('ullTotalVirtual', ctypes.c_ulonglong),
+      ('ullAvailVirtual', ctypes.c_ulonglong),
+      ('sullAvailExtendedVirtual', ctypes.c_ulonglong),
+  ]
+
+
+def get_total_memory():
+  if sys.platform in ('win32', 'cygwin'):
+    stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
+    success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
+    return stat.ullTotalPhys if success else 0
+  if sys.platform.startswith('linux'):
+    if os.path.exists('/proc/meminfo'):
+      with open('/proc/meminfo') as meminfo:
+        memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
+        for line in meminfo:
+          match = memtotal_re.match(line)
+          if match:
+            return float(match.group(1)) * 2**10
+  if sys.platform == 'darwin':
+    try:
+      return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
+    except:  # pylint: disable=bare-except
+      return 0
+  return 0
+
+
+def parse_size(string):
+  units = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40}
+  i = next(i for (i, c) in enumerate(string) if not c.isdigit())
+  number = string[:i].strip()
+  unit = string[i:].strip()
+  return int(float(number) * units[unit])
+
+
+def get_concurrent_jobs(reserve_memory, memory_per_job):
+  # reserve_memory = 2**30
+  total_memory = get_total_memory()
+  # Ensure the total memory used in the calculation below is at least 0
+  mem_total_bytes = max(0, total_memory - parse_size(reserve_memory))
+  # Ensure the number of cpus used in the calculation below is at least 1
+  try:
+    cpu_cap = multiprocessing.cpu_count()
+  except:  # pylint: disable=bare-except
+    cpu_cap = 1
+
+  # Calculate the number of jobs that will fit in memory. Ensure the
+  # value is at least 1.
+  num_concurrent_jobs = int(max(1, mem_total_bytes / parse_size(memory_per_job)))
+  # Cap the number of jobs by the number of cpus available.
+  concurrent_jobs = min(num_concurrent_jobs, cpu_cap)
+
+  return concurrent_jobs
+
+
 def to_gn_args(args):
   if args.simulator:
     if args.target_os != 'ios':
@@ -529,6 +599,8 @@ def to_gn_args(args):
   else:
     gn_args['dart_runtime_mode'] = runtime_mode
 
+  gn_args['concurrent_dart_jobs'] = get_concurrent_jobs('1GB', '1GB')
+
   # Desktop embeddings can have more dependencies than the engine library,
   # which can be problematic in some build environments (e.g., building on
   # Linux will bring in pkg-config dependencies at generation time). These
@@ -712,12 +784,12 @@ def to_gn_args(args):
   if args.prebuilt_impellerc is not None:
     gn_args['impeller_use_prebuilt_impellerc'] = args.prebuilt_impellerc
 
-  if args.malioc_path is not None:
-    gn_args['impeller_malioc_path'] = args.malioc_path
-  else:
+  malioc_path = args.malioc_path
+  if not malioc_path:
     malioc_path = os.environ.get('MALIOC_PATH')
-    if malioc_path:
-      gn_args['impeller_malioc_path'] = malioc_path
+  if malioc_path:
+    gn_args['impeller_malioc_path'] = malioc_path
+    gn_args['impeller_concurrent_malioc_jobs'] = get_concurrent_jobs('1GB', '100MB')
 
   if args.use_glfw_swiftshader:
     if get_host_os() == 'mac':