Merge remote-tracking branch 'origin/main' into workaround_windows_sh…

…ell_expansion_bug

emcc.py

@@ -2597,6 +2597,8 @@ def check_memory_setting(setting):
     # shared memory builds we must keep the memory segments in the wasm as
     # they will be passive segments which the .mem format cannot handle.
     settings.MEM_INIT_IN_WASM = not options.memory_init_file or settings.SINGLE_FILE or settings.SHARED_MEMORY
+  elif options.memory_init_file:
+    diagnostics.warning('unsupported', '--memory-init-file is only supported with -sWASM=0')
 
   if (
       settings.MAYBE_WASM2JS or
@@ -3544,8 +3546,6 @@ def consume_arg_file():
       ports.show_ports()
       should_exit = True
     elif check_arg('--memory-init-file'):
-      # This flag is ignored unless targetting wasm2js.
-      # TODO(sbc): Error out if used without wasm2js.
       options.memory_init_file = int(consume_arg())
     elif check_flag('--proxy-to-worker'):
       settings_changes.append('PROXY_TO_WORKER=1')

test/benchmark/benchmark_utf8.cpp → test/benchmark/benchmark_utf8.c

@@ -6,16 +6,18 @@
 #include <stdio.h>
 #include <string.h>
 #include <wchar.h>
-#include <iostream>
-#include <cassert>
+#include <assert.h>
 #include <emscripten.h>
+#include <time.h>
+
+EM_JS_DEPS(deps, "$UTF8ToString,emscripten_get_now");
 
 double test(const char *str) {
   double res = EM_ASM_DOUBLE({
     var t0 = _emscripten_get_now();
-    var str = Module.UTF8ToString($0);
+    var str = UTF8ToString($0);
     var t1 = _emscripten_get_now();
-//    out('t: ' + (t1 - t0) + ', len(result): ' + str.length + ', result: ' + str.slice(0, 100));
+    // out('t: ' + (t1 - t0) + ', len(result): ' + str.length + ', result: ' + str.slice(0, 100));
     return (t1-t0);
   }, str);
   return res;
@@ -30,38 +32,40 @@ char *randomString(int len) {
     fseek(handle, 0, SEEK_END);
     utf8_corpus_length = ftell(handle);
     assert(utf8_corpus_length > 0);
-    utf8_corpus = new char[utf8_corpus_length+1];
+    utf8_corpus = malloc(utf8_corpus_length+1);
     fseek(handle, 0, SEEK_SET);
     fread(utf8_corpus, 1, utf8_corpus_length, handle);
     fclose(handle);
     utf8_corpus[utf8_corpus_length] = '\0';
   }
   int startIdx = rand() % (utf8_corpus_length - len);
-  while(((unsigned char)utf8_corpus[startIdx] & 0xC0) == 0x80) {
+  while (((unsigned char)utf8_corpus[startIdx] & 0xC0) == 0x80) {
     ++startIdx;
     if (startIdx + len > utf8_corpus_length) len = utf8_corpus_length - startIdx;
   }
   assert(len > 0);
-  char *s = new char[len+1];
+  char *s = malloc(len+1);
   memcpy(s, utf8_corpus + startIdx, len);
   s[len] = '\0';
-  while(len > 0 && ((unsigned char)s[len-1] & 0xC0) == 0x80) { s[--len] = '\0'; }
-  while(len > 0 && ((unsigned char)s[len-1] & 0xC0) == 0xC0) { s[--len] = '\0'; }
+  while (len > 0 && ((unsigned char)s[len-1] & 0xC0) == 0x80) { s[--len] = '\0'; }
+  while (len > 0 && ((unsigned char)s[len-1] & 0xC0) == 0xC0) { s[--len] = '\0'; }
   assert(len >= 0);
   return s;
 }
 
 int main() {
-  srand(time(NULL));
+  time_t seed = time(NULL);
+  printf("Random seed: %lld\n", seed);
+  srand(seed);
   double t = 0;
   double t2 = emscripten_get_now();
-  for(int i = 0; i < 100000; ++i) {
+  for (int i = 0; i < 100000; ++i) {
     // Create strings of lengths 1-32, because the internals of text decoding
     // have a cutoff of 16 for when to use TextDecoder, and we wish to test both
     // (see UTF8ArrayToString).
     char *str = randomString((i % 32) + 1);
     t += test(str);
-    delete [] str;
+    free(str);
   }
   double t3 = emscripten_get_now();
   printf("OK. Time: %f (%f).\n", t, t3-t2);

test/other/metadce/test_metadce_minimal_pthreads.jssize

@@ -1 +1 @@
-15562
+15528

test/parallel_testsuite.py

@@ -12,6 +12,8 @@
 
 import common
 
+from tools.shared import cap_max_workers_in_pool
+
 
 NUM_CORES = None
 
@@ -52,7 +54,7 @@ def run(self, result):
     # issues.
     # multiprocessing.set_start_method('spawn')
     tests = list(self.reversed_tests())
-    use_cores = min(self.max_cores, len(tests), num_cores())
+    use_cores = cap_max_workers_in_pool(min(self.max_cores, len(tests), num_cores()))
     print('Using %s parallel test processes' % use_cores)
     pool = multiprocessing.Pool(use_cores)
     results = [pool.apply_async(run_test, (t,)) for t in tests]

test/test_browser.py

@@ -743,13 +743,14 @@ def test_sdl_image(self):
     shutil.copyfile(test_file('screenshot.jpg'), 'screenshot.jpg')
     src = test_file('browser/test_sdl_image.c')
 
-    for mem in [0, 1]:
+    for args in [[], ['--memory-init-file=1', '-sWASM=0']]:
       for dest, dirname, basename in [('screenshot.jpg', '/', 'screenshot.jpg'),
                                       ('screenshot.jpg@/assets/screenshot.jpg', '/assets', 'screenshot.jpg')]:
-        self.btest_exit(src, args=[
-          '-O2', '-lSDL', '-lGL', '--memory-init-file', str(mem),
+        self.btest_exit(src, args=args + [
+          '-O2', '-lSDL', '-lGL',
           '--preload-file', dest, '-DSCREENSHOT_DIRNAME="' + dirname + '"', '-DSCREENSHOT_BASENAME="' + basename + '"', '--use-preload-plugins'
         ])
+      return
 
   @also_with_wasmfs
   def test_sdl_image_jpeg(self):
@@ -1485,11 +1486,11 @@ def test_idbstore(self):
   def test_idbstore_sync(self):
     secret = str(time.time())
     self.clear()
-    self.btest(test_file('idbstore_sync.c'), '6', args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '--memory-init-file', '1', '-O3', '-g2', '-sASYNCIFY'])
+    self.btest(test_file('idbstore_sync.c'), '6', args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '-O3', '-g2', '-sASYNCIFY'])
 
   def test_idbstore_sync_worker(self):
     secret = str(time.time())
-    self.btest(test_file('idbstore_sync_worker.c'), expected='0', args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '--memory-init-file', '1', '-O3', '-g2', '--proxy-to-worker', '-sINITIAL_MEMORY=80MB', '-sASYNCIFY'])
+    self.btest(test_file('idbstore_sync_worker.c'), expected='0', args=['-lidbstore.js', '-DSECRET=\"' + secret + '\"', '-O3', '-g2', '--proxy-to-worker', '-sINITIAL_MEMORY=80MB', '-sASYNCIFY'])
 
   def test_force_exit(self):
     self.btest_exit('force_exit.c', assert_returncode=10)
@@ -2347,8 +2348,8 @@ def test_pre_run_deps(self):
       };
     ''')
 
-    for mem in [0, 1]:
-      self.btest('pre_run_deps.cpp', expected='10', args=['--pre-js', 'pre.js', '--memory-init-file', str(mem)])
+    for args in [[], ['-sWASM=0', '--memory-init-file=1']]:
+      self.btest('pre_run_deps.cpp', expected='10', args=args + ['--pre-js', 'pre.js'])
 
   def test_mem_init(self):
     self.set_setting('WASM_ASYNC_COMPILATION', 0)
@@ -2398,7 +2399,7 @@ def test(what, status):
 
   @parameterized({
     '': ([],),
-    'asmjs': (['-sWASM=0'],)
+    'asmjs': (['-sWASM=0', '--memory-init-file=1'],)
   })
   def test_runtime_misuse(self, mode):
     self.set_setting('DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', '$ccall,$cwrap')
@@ -2483,7 +2484,7 @@ def test_runtime_misuse(self, mode):
 
       print('mem init, so async, call too early')
       create_file('post.js', post_prep + post_test + post_hook)
-      self.btest(filename, expected='600', args=['--post-js', 'post.js', '--memory-init-file', '1', '-sEXIT_RUNTIME'] + extra_args + mode, reporting=Reporting.NONE)
+      self.btest(filename, expected='600', args=['--post-js', 'post.js', '-sEXIT_RUNTIME'] + extra_args + mode, reporting=Reporting.NONE)
       print('sync startup, call too late')
       create_file('post.js', post_prep + 'Module.postRun.push(function() { ' + post_test + ' });' + post_hook)
       self.btest(filename, expected=str(second_code), args=['--post-js', 'post.js', '-sEXIT_RUNTIME'] + extra_args + mode, reporting=Reporting.NONE)
@@ -2902,11 +2903,11 @@ def test_sdl2_image(self):
     # load an image file, get pixel data. Also O2 coverage for --preload-file, and memory-init
     shutil.copyfile(test_file('screenshot.jpg'), 'screenshot.jpg')
 
-    for mem in [0, 1]:
+    for args in [[], ['--memory-init-file=1', '-sWASM=0']]:
       for dest, dirname, basename in [('screenshot.jpg', '/', 'screenshot.jpg'),
                                       ('screenshot.jpg@/assets/screenshot.jpg', '/assets', 'screenshot.jpg')]:
-        self.btest_exit('browser/test_sdl2_image.c', 600, args=[
-          '-O2', '--memory-init-file', str(mem),
+        self.btest_exit('browser/test_sdl2_image.c', 600, args=args + [
+          '-O2',
           '--preload-file', dest,
           '-DSCREENSHOT_DIRNAME="' + dirname + '"',
           '-DSCREENSHOT_BASENAME="' + basename + '"',
@@ -4137,16 +4138,19 @@ def test_pthread_call_async_on_main_thread(self):
   # Tests that spawning a new thread does not cause a reinitialization of the global data section of the application memory area.
   @requires_threads
   def test_pthread_global_data_initialization(self):
-    mem_init_modes = [[], ['--memory-init-file', '0'], ['--memory-init-file', '1']]
+    # --memory-init-file mode disabled because WASM=0 does not seem to work with EXPORT_NAME
+    mem_init_modes = [[]]  # ['-sWASM=0', '--memory-init-file', '0'], ['-sWASM=0', '--memory-init-file', '1']]
     for mem_init_mode in mem_init_modes:
+      print(mem_init_mode)
       for args in [['-sMODULARIZE', '-sEXPORT_NAME=MyModule', '--shell-file', test_file('shell_that_launches_modularize.html')], ['-O3']]:
         self.btest_exit(test_file('pthread/test_pthread_global_data_initialization.c'), args=args + mem_init_mode + ['-pthread', '-sPROXY_TO_PTHREAD', '-sPTHREAD_POOL_SIZE'])
 
   @requires_threads
   @requires_sync_compilation
   def test_pthread_global_data_initialization_in_sync_compilation_mode(self):
-    mem_init_modes = [[], ['--memory-init-file', '0'], ['--memory-init-file', '1']]
+    mem_init_modes = [[], ['-sWASM=0', '--memory-init-file', '0'], ['-sWASM=0', '--memory-init-file', '1']]
     for mem_init_mode in mem_init_modes:
+      print(mem_init_mode)
       args = ['-sWASM_ASYNC_COMPILATION=0']
       self.btest_exit(test_file('pthread/test_pthread_global_data_initialization.c'), args=args + mem_init_mode + ['-pthread', '-sPROXY_TO_PTHREAD', '-sPTHREAD_POOL_SIZE'])
 
@@ -4363,7 +4367,7 @@ def test_wasm_locate_file(self):
 
   @also_with_threads
   def test_utf8_textdecoder(self):
-    self.btest_exit(test_file('benchmark/benchmark_utf8.cpp'), 0, args=['--embed-file', test_file('utf8_corpus.txt') + '@/utf8_corpus.txt', '-sEXPORTED_RUNTIME_METHODS=[UTF8ToString]'])
+    self.btest_exit(test_file('benchmark/benchmark_utf8.c'), 0, args=['--embed-file', test_file('utf8_corpus.txt') + '@/utf8_corpus.txt'])
 
   @also_with_threads
   def test_utf16_textdecoder(self):

test/test_core.py

@@ -5896,9 +5896,8 @@ def test_utf8(self):
 
   @also_with_wasm_bigint
   def test_utf8_textdecoder(self):
-    self.set_setting('EXPORTED_RUNTIME_METHODS', ['UTF8ToString', 'stringToUTF8'])
     self.emcc_args += ['--embed-file', test_file('utf8_corpus.txt') + '@/utf8_corpus.txt']
-    self.do_runf(test_file('benchmark/benchmark_utf8.cpp'), 'OK.')
+    self.do_runf(test_file('benchmark/benchmark_utf8.c'), 'OK.')
 
   # Test that invalid character in UTF8 does not cause decoding to crash.
   def test_utf8_invalid(self):

test/test_other.py

@@ -141,7 +141,7 @@ def decorated(self, *args, **kwargs):
       if 'EMTEST_SKIP_SCONS' in os.environ:
         self.skipTest('test requires scons and EMTEST_SKIP_SCONS is set')
       else:
-        self.fail('scons required to run this test.  Use EMTEST_SKIP_SKIP to skip')
+        self.fail('scons required to run this test.  Use EMTEST_SKIP_SCONS to skip')
     return func(self, *args, **kwargs)
 
   return decorated
@@ -13375,7 +13375,12 @@ def run(args, expect_bulk_mem):
     self.setup_node_pthreads()
     run(['-pthread'], expect_bulk_mem=True)
 
+  def test_memory_init_file_unsupported(self):
+    err = self.expect_fail([EMCC, test_file('hello_world.c'), '-Werror', '--memory-init-file=1'])
+    self.assertContained('error: --memory-init-file is only supported with -sWASM=0 [-Wunsupported] [-Werror]', err)
+
   @only_windows('This test verifies Windows batch script behavior against bug https://github.com/microsoft/terminal/issues/15212')
   def test_windows_batch_file_dp0_expansion_bug(self):
     open('build_with_quotes.bat', 'w').write(f'@"emcc" {test_file("hello_world.c")}')
     self.run_process(['build_with_quotes.bat'])
+

tools/shared.py

@@ -78,6 +78,7 @@
 diagnostics.add_warning('em-js-i64')
 diagnostics.add_warning('js-compiler')
 diagnostics.add_warning('compatibility')
+diagnostics.add_warning('unsupported')
 # Closure warning are not (yet) enabled by default
 diagnostics.add_warning('closure', enabled=False)
 
@@ -142,6 +143,13 @@ def returncode_to_str(code):
   return f'returned {code}'
 
 
+def cap_max_workers_in_pool(max_workers):
+  # Python has an issue that it can only use max 61 cores on Windows: https://github.com/python/cpython/issues/89240
+  if WINDOWS:
+    return min(max_workers, 61)
+  return max_workers
+
+
 def run_multiple_processes(commands,
                            env=None,
                            route_stdout_to_temp_files_suffix=None):
@@ -161,13 +169,9 @@ def run_multiple_processes(commands,
   # faster, but may not work on Windows.
   if int(os.getenv('EM_PYTHON_MULTIPROCESSING', '0')):
     import multiprocessing
-    max_workers = get_num_cores()
     global multiprocessing_pool
     if not multiprocessing_pool:
-      if WINDOWS:
-        # Fix for python < 3.8 on windows. See: https://github.com/python/cpython/pull/13132
-        max_workers = min(max_workers, 61)
-      multiprocessing_pool = multiprocessing.Pool(processes=max_workers)
+      multiprocessing_pool = multiprocessing.Pool(processes=cap_max_workers_in_pool(get_num_cores()))
     return multiprocessing_pool.map(mp_run_process, [(cmd, env, route_stdout_to_temp_files_suffix) for cmd in commands], chunksize=1)
 
   std_outs = []