From 457e0bd4faedd30d138160cb99cfb2387994fd93 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Fri, 13 Dec 2024 18:16:49 +0100 Subject: [PATCH] Auto-format Python code with ruff format --- .circleci/config.yml | 1 + .style.yapf | 3 - bootstrap.py | 22 +- docs/process.md | 5 +- em-config.py | 4 +- emar.py | 3 +- embuilder.py | 212 ++++---- emcc.py | 244 ++++++--- emcmake.py | 12 +- emconfigure.py | 7 +- emmake.py | 7 +- emrun.py | 681 ++++++++++++++++--------- emsize.py | 3 +- emstrip.py | 3 +- emsymbolizer.py | 50 +- pyproject.toml | 6 + requirements-dev.txt | 2 +- site/source/get_api_items.py | 132 ++--- site/source/get_wiki.py | 322 ++++++------ system/bin/sdl-config.py | 1 - system/bin/sdl2-config.py | 1 - test/benchmark/benchmark_sse.py | 372 ++++++++------ test/clang_native.py | 38 +- test/common.py | 389 ++++++++------ test/gen_large_switchcase.py | 14 +- test/gen_many_js_functions.py | 13 +- test/jsrun.py | 28 +- test/other/ports/external.py | 7 +- test/parallel_testsuite.py | 13 +- test/runner.py | 76 +-- test/test_benchmark.py | 564 ++++++++++++++------ test/test_interactive.py | 364 ++++++++++--- test/test_posixtest.py | 29 +- test/test_posixtest_browser.py | 1 + test/test_sanity.py | 101 ++-- test/test_sockets.py | 194 ++++--- tools/building.py | 140 ++--- tools/cache.py | 19 +- tools/clean_webconsole.py | 15 +- tools/colored_logger.py | 90 ++-- tools/config.py | 14 +- tools/config_template.py | 6 +- tools/create_dom_pk_codes.py | 341 +++++++------ tools/determinism_checker.py | 5 +- tools/diagnostics.py | 42 +- tools/emcoverage.py | 2 +- tools/emdump.py | 340 +++++++++---- tools/emdwp.py | 3 +- tools/emnm.py | 3 +- tools/emprofile.py | 11 +- tools/experimental/reproduceriter.py | 24 +- tools/extract_metadata.py | 29 +- tools/feature_matrix.py | 9 +- tools/file_packager.py | 163 +++--- tools/install.py | 12 +- tools/js_manipulation.py | 16 +- tools/js_optimizer.py | 71 ++- tools/line_endings.py | 49 +- tools/maint/add_license.py | 28 +- tools/maint/create_entry_points.py | 4 +- tools/maint/create_release.py | 7 +- tools/maint/gen_sig_info.py | 133 +++-- tools/maint/gen_struct_info.py | 110 ++-- tools/maint/simde_update.py | 8 +- tools/maybe_wasm2js.py | 3 +- tools/minimal_runtime_shell.py | 55 +- tools/ports/__init__.py | 25 +- tools/ports/boost_headers.py | 8 +- tools/ports/bullet.py | 5 +- tools/ports/bzip2.py | 9 +- tools/ports/cocos2d.py | 68 +-- tools/ports/contrib/glfw3.py | 24 +- tools/ports/freetype.py | 91 ++-- tools/ports/giflib.py | 25 +- tools/ports/harfbuzz.py | 6 +- tools/ports/icu.py | 48 +- tools/ports/libjpeg.py | 19 +- tools/ports/libpng.py | 8 +- tools/ports/mpg123.py | 4 +- tools/ports/regal.py | 112 ++-- tools/ports/sdl2_image.py | 37 +- tools/ports/sdl2_mixer.py | 6 +- tools/ports/sdl2_ttf.py | 2 +- tools/ports/vorbis.py | 10 +- tools/response_file.py | 19 +- tools/settings.py | 189 +++---- tools/shared.py | 43 +- tools/system_libs.py | 735 +++++++++++++++------------ tools/tempfiles.py | 4 +- tools/toolchain_profiler.py | 95 +++- tools/utils.py | 2 +- tools/wasm-sourcemap.py | 59 ++- tools/webassembly.py | 66 +-- tools/webidl_binder.py | 368 ++++++++++---- 94 files changed, 4860 insertions(+), 2903 deletions(-) delete mode 100644 .style.yapf diff --git a/.circleci/config.yml b/.circleci/config.yml index 61d7cb912a278..42fb755710d70 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -427,6 +427,7 @@ jobs: - run: ruff check # TODO (cclauss): When ruff supports rule E303 without --preview, remove following line - run: ruff check --preview --select=E303 + - run: ruff format mypy: executor: bionic steps: diff --git a/.style.yapf b/.style.yapf deleted file mode 100644 index de72b98c2c862..0000000000000 --- a/.style.yapf +++ /dev/null @@ -1,3 +0,0 @@ -[style] -based_on_style = pep8 -indent_width = 2 diff --git a/bootstrap.py b/bootstrap.py index ff9d55dc129a2..bfd95fb7b5b3a 100755 --- a/bootstrap.py +++ b/bootstrap.py @@ -7,6 +7,7 @@ on the timestamps of various input files (kind of like a dumb version of a Makefile). """ + import argparse import os import shutil @@ -21,13 +22,16 @@ actions = [ ('npm packages', ['package.json'], [shutil.which('npm'), 'ci']), - ('create entry points', [ - 'tools/maint/create_entry_points.py', - 'tools/maint/run_python.bat', - 'tools/maint/run_python.sh', - 'tools/maint/run_python.ps1', - ], - [sys.executable, 'tools/maint/create_entry_points.py']), + ( + 'create entry points', + [ + 'tools/maint/create_entry_points.py', + 'tools/maint/run_python.bat', + 'tools/maint/run_python.sh', + 'tools/maint/run_python.ps1', + ], + [sys.executable, 'tools/maint/create_entry_points.py'], + ), ('git submodules', ['test/third_party/posixtestsuite/'], [shutil.which('git'), 'submodule', 'update', '--init']), ] @@ -57,7 +61,9 @@ def main(args): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('-v', '--verbose', action='store_true', help='verbose', default=False) parser.add_argument('-n', '--dry-run', action='store_true', help='dry run', default=False) - parser.add_argument('-i', '--install-post-checkout', action='store_true', help='install post checkout script', default=False) + parser.add_argument( + '-i', '--install-post-checkout', action='store_true', help='install post checkout script', default=False + ) args = parser.parse_args() if args.install_post_checkout: diff --git a/docs/process.md b/docs/process.md index f1bf04a714a11..5a5a7f2ff8243 100644 --- a/docs/process.md +++ b/docs/process.md @@ -55,8 +55,9 @@ pre-processor. See [`.clang-format`][clang-format] for more details. ### Python Code We generally follow the pep8 standard with the major exception that we use 2 -spaces for indentation. `ruff` is run on all PRs to ensure that Python code -conforms to this style. See [`pyproject.toml`][pyproject.toml] for more details. +spaces for indentation. `ruff check` and `ruff format` are run on all PRs to +ensure that Python code conforms to this style. See +[`pyproject.toml`][pyproject.toml] for more details. #### Static Type Checking diff --git a/em-config.py b/em-config.py index bcbd9abe1cf8d..73e4e009bfd47 100755 --- a/em-config.py +++ b/em-config.py @@ -20,9 +20,7 @@ def main(): - if len(sys.argv) != 2 or \ - not re.match(r"^[\w\W_][\w\W_\d]*$", sys.argv[1]) or \ - not hasattr(config, sys.argv[1]): + if len(sys.argv) != 2 or not re.match(r"^[\w\W_][\w\W_\d]*$", sys.argv[1]) or not hasattr(config, sys.argv[1]): print('Usage: em-config VAR_NAME', file=sys.stderr) sys.exit(1) diff --git a/emar.py b/emar.py index 061fc5d4ce927..88625b01407c1 100755 --- a/emar.py +++ b/emar.py @@ -4,8 +4,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Wrapper script around `llvm-ar`. -""" +"""Wrapper script around `llvm-ar`.""" import sys from tools import shared diff --git a/embuilder.py b/embuilder.py index 4b706c1d6df76..e5b426c05a551 100755 --- a/embuilder.py +++ b/embuilder.py @@ -30,96 +30,96 @@ # Minimal subset of targets used by CI systems to build enough to be useful MINIMAL_TASKS = [ - 'libbulkmemory', - 'libcompiler_rt', - 'libcompiler_rt-wasm-sjlj', - 'libcompiler_rt-ww', - 'libc', - 'libc-debug', - 'libc-ww-debug', - 'libc_optz', - 'libc_optz-debug', - 'libc++abi', - 'libc++abi-except', - 'libc++abi-noexcept', - 'libc++abi-debug', - 'libc++abi-debug-except', - 'libc++abi-debug-noexcept', - 'libc++abi-debug-ww-noexcept', - 'libc++', - 'libc++-except', - 'libc++-noexcept', - 'libc++-ww-noexcept', - 'libal', - 'libdlmalloc', - 'libdlmalloc-tracing', - 'libdlmalloc-debug', - 'libdlmalloc-ww', - 'libembind', - 'libembind-rtti', - 'libemmalloc', - 'libemmalloc-debug', - 'libemmalloc-memvalidate', - 'libemmalloc-verbose', - 'libemmalloc-memvalidate-verbose', - 'libmimalloc', - 'libmimalloc-mt', - 'libGL', - 'libGL-getprocaddr', - 'libGL-emu-getprocaddr', - 'libGL-emu-webgl2-ofb-getprocaddr', - 'libGL-webgl2-ofb-getprocaddr', - 'libGL-ww-getprocaddr', - 'libhtml5', - 'libsockets', - 'libsockets-ww', - 'libstubs', - 'libstubs-debug', - 'libstandalonewasm-nocatch', - 'crt1', - 'crt1_proxy_main', - 'crtbegin', - 'libunwind-except', - 'libnoexit', - 'sqlite3', - 'sqlite3-mt', - 'libwebgpu', - 'libwebgpu_cpp', + 'libbulkmemory', + 'libcompiler_rt', + 'libcompiler_rt-wasm-sjlj', + 'libcompiler_rt-ww', + 'libc', + 'libc-debug', + 'libc-ww-debug', + 'libc_optz', + 'libc_optz-debug', + 'libc++abi', + 'libc++abi-except', + 'libc++abi-noexcept', + 'libc++abi-debug', + 'libc++abi-debug-except', + 'libc++abi-debug-noexcept', + 'libc++abi-debug-ww-noexcept', + 'libc++', + 'libc++-except', + 'libc++-noexcept', + 'libc++-ww-noexcept', + 'libal', + 'libdlmalloc', + 'libdlmalloc-tracing', + 'libdlmalloc-debug', + 'libdlmalloc-ww', + 'libembind', + 'libembind-rtti', + 'libemmalloc', + 'libemmalloc-debug', + 'libemmalloc-memvalidate', + 'libemmalloc-verbose', + 'libemmalloc-memvalidate-verbose', + 'libmimalloc', + 'libmimalloc-mt', + 'libGL', + 'libGL-getprocaddr', + 'libGL-emu-getprocaddr', + 'libGL-emu-webgl2-ofb-getprocaddr', + 'libGL-webgl2-ofb-getprocaddr', + 'libGL-ww-getprocaddr', + 'libhtml5', + 'libsockets', + 'libsockets-ww', + 'libstubs', + 'libstubs-debug', + 'libstandalonewasm-nocatch', + 'crt1', + 'crt1_proxy_main', + 'crtbegin', + 'libunwind-except', + 'libnoexit', + 'sqlite3', + 'sqlite3-mt', + 'libwebgpu', + 'libwebgpu_cpp', ] # Additional tasks on top of MINIMAL_TASKS that are necessary for PIC testing on # CI (which has slightly more tests than other modes that want to use MINIMAL) MINIMAL_PIC_TASKS = MINIMAL_TASKS + [ - 'libcompiler_rt-mt', - 'libc-mt', - 'libc-mt-debug', - 'libc_optz-mt', - 'libc_optz-mt-debug', - 'libc++abi-mt', - 'libc++abi-mt-noexcept', - 'libc++abi-debug-mt', - 'libc++abi-debug-mt-noexcept', - 'libc++-mt', - 'libc++-mt-noexcept', - 'libdlmalloc-mt', - 'libGL-emu', - 'libGL-emu-webgl2-getprocaddr', - 'libGL-mt-getprocaddr', - 'libGL-mt-emu', - 'libGL-mt-emu-webgl2-getprocaddr', - 'libGL-mt-emu-webgl2-ofb-getprocaddr', - 'libsockets_proxy', - 'libsockets-mt', - 'crtbegin', - 'libsanitizer_common_rt', - 'libubsan_rt', - 'libwasm_workers-debug-stub', - 'libfetch', - 'libfetch-mt', - 'libwasmfs', - 'libwasmfs-debug', - 'libwasmfs_no_fs', - 'giflib', + 'libcompiler_rt-mt', + 'libc-mt', + 'libc-mt-debug', + 'libc_optz-mt', + 'libc_optz-mt-debug', + 'libc++abi-mt', + 'libc++abi-mt-noexcept', + 'libc++abi-debug-mt', + 'libc++abi-debug-mt-noexcept', + 'libc++-mt', + 'libc++-mt-noexcept', + 'libdlmalloc-mt', + 'libGL-emu', + 'libGL-emu-webgl2-getprocaddr', + 'libGL-mt-getprocaddr', + 'libGL-mt-emu', + 'libGL-mt-emu-webgl2-getprocaddr', + 'libGL-mt-emu-webgl2-ofb-getprocaddr', + 'libsockets_proxy', + 'libsockets-mt', + 'crtbegin', + 'libsanitizer_common_rt', + 'libubsan_rt', + 'libwasm_workers-debug-stub', + 'libfetch', + 'libfetch-mt', + 'libwasmfs', + 'libwasmfs-debug', + 'libwasmfs_no_fs', + 'giflib', ] PORTS = sorted(list(ports.ports_by_name.keys()) + list(ports.port_variants.keys())) @@ -187,19 +187,17 @@ def handle_port_error(target, message): def main(): all_build_start_time = time.time() - parser = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=get_help()) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter, epilog=get_help() + ) parser.add_argument('--lto', action='store_const', const='full', help='build bitcode object for LTO') - parser.add_argument('--lto=thin', dest='lto', action='store_const', const='thin', help='build bitcode object for ThinLTO') - parser.add_argument('--pic', action='store_true', - help='build relocatable objects for suitable for dynamic linking') - parser.add_argument('--force', action='store_true', - help='force rebuild of target (by removing it first)') - parser.add_argument('--verbose', action='store_true', - help='show build commands') - parser.add_argument('--wasm64', action='store_true', - help='use wasm64 architecture') + parser.add_argument( + '--lto=thin', dest='lto', action='store_const', const='thin', help='build bitcode object for ThinLTO' + ) + parser.add_argument('--pic', action='store_true', help='build relocatable objects for suitable for dynamic linking') + parser.add_argument('--force', action='store_true', help='force rebuild of target (by removing it first)') + parser.add_argument('--verbose', action='store_true', help='show build commands') + parser.add_argument('--wasm64', action='store_true', help='use wasm64 architecture') parser.add_argument('operation', choices=['build', 'clear', 'rebuild']) parser.add_argument('targets', nargs='*', help='see below') args = parser.parse_args() @@ -240,7 +238,7 @@ def main(): # process tasks auto_tasks = False - task_targets = dict.fromkeys(args.targets) # use dict to keep targets order + task_targets = dict.fromkeys(args.targets) # use dict to keep targets order # substitute predefined_tasks = { @@ -315,14 +313,28 @@ def main(): return 1 time_taken = time.time() - start_time - logger.info('...success. Took %s(%.2fs)' % (('%02d:%02d mins ' % (time_taken // 60, time_taken % 60) if time_taken >= 60 else ''), time_taken)) + logger.info( + '...success. Took %s(%.2fs)' + % (('%02d:%02d mins ' % (time_taken // 60, time_taken % 60) if time_taken >= 60 else ''), time_taken) + ) if USE_NINJA and args.operation != 'clear': system_libs.build_deferred() if len(tasks) > 1 or USE_NINJA: all_build_time_taken = time.time() - all_build_start_time - logger.info('Built %d targets in %s(%.2fs)' % (len(tasks), ('%02d:%02d mins ' % (all_build_time_taken // 60, all_build_time_taken % 60) if all_build_time_taken >= 60 else ''), all_build_time_taken)) + logger.info( + 'Built %d targets in %s(%.2fs)' + % ( + len(tasks), + ( + '%02d:%02d mins ' % (all_build_time_taken // 60, all_build_time_taken % 60) + if all_build_time_taken >= 60 + else '' + ), + all_build_time_taken, + ) + ) return 0 diff --git a/emcc.py b/emcc.py index b519281e72e84..e8023444db37c 100644 --- a/emcc.py +++ b/emcc.py @@ -56,6 +56,7 @@ # run already. if os.path.exists(utils.path_from_root('.git')) and os.path.exists(utils.path_from_root('bootstrap.py')): import bootstrap + bootstrap.check() # endings = dot + a suffix, compare against result of shared.suffix() @@ -65,7 +66,7 @@ PREPROCESSED_ENDINGS = ['.i', '.ii'] OBJCXX_ENDINGS = ['.mm', '.mii'] SPECIAL_ENDINGLESS_FILENAMES = [os.devnull] -C_ENDINGS += SPECIAL_ENDINGLESS_FILENAMES # consider the special endingless filenames like /dev/null to be C +C_ENDINGS += SPECIAL_ENDINGLESS_FILENAMES # consider the special endingless filenames like /dev/null to be C SOURCE_ENDINGS = C_ENDINGS + CXX_ENDINGS + OBJC_ENDINGS + OBJCXX_ENDINGS + ['.bc', '.ll', '.S'] ASSEMBLY_ENDINGS = ['.s'] @@ -73,20 +74,34 @@ # These symbol names are allowed in INCOMING_MODULE_JS_API but are not part of the # default set. -EXTRA_INCOMING_JS_API = [ - 'fetchSettings' -] +EXTRA_INCOMING_JS_API = ['fetchSettings'] SIMD_INTEL_FEATURE_TOWER = ['-msse', '-msse2', '-msse3', '-mssse3', '-msse4.1', '-msse4.2', '-msse4', '-mavx'] SIMD_NEON_FLAGS = ['-mfpu=neon'] LINK_ONLY_FLAGS = { - '--bind', '--closure', '--cpuprofiler', '--embed-file', - '--emit-symbol-map', '--emrun', '--exclude-file', '--extern-post-js', - '--extern-pre-js', '--ignore-dynamic-linking', '--js-library', - '--js-transform', '--oformat', '--output_eol', - '--post-js', '--pre-js', '--preload-file', '--profiling-funcs', - '--proxy-to-worker', '--shell-file', '--source-map-base', - '--threadprofiler', '--use-preload-plugins' + '--bind', + '--closure', + '--cpuprofiler', + '--embed-file', + '--emit-symbol-map', + '--emrun', + '--exclude-file', + '--extern-post-js', + '--extern-pre-js', + '--ignore-dynamic-linking', + '--js-library', + '--js-transform', + '--oformat', + '--output_eol', + '--post-js', + '--pre-js', + '--preload-file', + '--profiling-funcs', + '--proxy-to-worker', + '--shell-file', + '--source-map-base', + '--threadprofiler', + '--use-preload-plugins', } @@ -144,10 +159,10 @@ def __init__(self): self.use_closure_compiler = None self.closure_args = [] self.js_transform = None - self.pre_js = [] # before all js - self.post_js = [] # after all js - self.extern_pre_js = [] # before all js, external to optimized code - self.extern_post_js = [] # after all js, external to optimized code + self.pre_js = [] # before all js + self.post_js = [] # after all js + self.extern_pre_js = [] # before all js, external to optimized code + self.extern_post_js = [] # after all js, external to optimized code self.preload_files = [] self.embed_files = [] self.exclude_files = [] @@ -218,13 +233,36 @@ def make_relative(filename): if ignore: continue - if arg in ('-MT', '-MF', '-MJ', '-MQ', '-D', '-U', '-o', '-x', - '-Xpreprocessor', '-include', '-imacros', '-idirafter', - '-iprefix', '-iwithprefix', '-iwithprefixbefore', - '-isysroot', '-imultilib', '-A', '-isystem', '-iquote', - '-install_name', '-compatibility_version', - '-current_version', '-I', '-L', '-include-pch', - '-Xlinker', '-Xclang'): + if arg in ( + '-MT', + '-MF', + '-MJ', + '-MQ', + '-D', + '-U', + '-o', + '-x', + '-Xpreprocessor', + '-include', + '-imacros', + '-idirafter', + '-iprefix', + '-iwithprefix', + '-iwithprefixbefore', + '-isysroot', + '-imultilib', + '-A', + '-isystem', + '-iquote', + '-install_name', + '-compatibility_version', + '-current_version', + '-I', + '-L', + '-include-pch', + '-Xlinker', + '-Xclang', + ): ignore_next = True if arg == '-o': @@ -244,7 +282,7 @@ def expand_byte_size_suffixes(value): value, suffix = match.groups() value = int(value) if suffix: - size_suffixes = {suffix: 1024 ** i for i, suffix in enumerate(['b', 'kb', 'mb', 'gb', 'tb'])} + size_suffixes = {suffix: 1024**i for i, suffix in enumerate(['b', 'kb', 'mb', 'gb', 'tb'])} value *= size_suffixes[suffix.lower()] return value @@ -472,7 +510,11 @@ def array_contains_any_of(hay, needles): if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER) or array_contains_any_of(user_args, SIMD_NEON_FLAGS): if '-msimd128' not in user_args and '-mrelaxed-simd' not in user_args: - exit_with_error('passing any of ' + ', '.join(SIMD_INTEL_FEATURE_TOWER + SIMD_NEON_FLAGS) + ' flags also requires passing -msimd128 (or -mrelaxed-simd)!') + exit_with_error( + 'passing any of ' + + ', '.join(SIMD_INTEL_FEATURE_TOWER + SIMD_NEON_FLAGS) + + ' flags also requires passing -msimd128 (or -mrelaxed-simd)!' + ) cflags += ['-D__SSE__=1'] if array_contains_any_of(user_args, SIMD_INTEL_FEATURE_TOWER[1:]): @@ -588,7 +630,7 @@ def run(args): ''') return 0 - if '-dumpversion' in args: # gcc's doc states "Print the compiler version [...] and don't do anything else." + if '-dumpversion' in args: # gcc's doc states "Print the compiler version [...] and don't do anything else." print(utils.EMSCRIPTEN_VERSION) return 0 @@ -609,16 +651,22 @@ def run(args): if not lines: exit_with_error(f'unable to parse output of `{cmd}`:\n{proc.stderr}') parts = shlex.split(lines[0].replace('\\', '\\\\')) - parts = [x for x in parts if x not in ['-c', '-o', '-v', '-emit-llvm'] and input_file not in x and temp_target not in x] + parts = [ + x for x in parts if x not in ['-c', '-o', '-v', '-emit-llvm'] and input_file not in x and temp_target not in x + ] print(shared.shlex_join(parts[1:])) return 0 if 'EMMAKEN_NO_SDK' in os.environ: - exit_with_error('EMMAKEN_NO_SDK is no longer supported. The standard -nostdlib and -nostdinc flags should be used instead') + exit_with_error( + 'EMMAKEN_NO_SDK is no longer supported. The standard -nostdlib and -nostdinc flags should be used instead' + ) if 'EMMAKEN_COMPILER' in os.environ: - exit_with_error('`EMMAKEN_COMPILER` is no longer supported.\n' + - 'Please use the `LLVM_ROOT` and/or `COMPILER_WRAPPER` config settings instead') + exit_with_error( + '`EMMAKEN_COMPILER` is no longer supported.\n' + + 'Please use the `LLVM_ROOT` and/or `COMPILER_WRAPPER` config settings instead' + ) if 'EMMAKEN_CFLAGS' in os.environ: exit_with_error('`EMMAKEN_CFLAGS` is no longer supported, please use `EMCC_CFLAGS` instead') @@ -670,6 +718,7 @@ def run(args): exit_with_error('--post-link requires a single input file') # Delay import of link.py to avoid processing this file when only compiling from tools import link + link.run_post_link(input_files[0][1], options, state, newargs) return 0 @@ -679,6 +728,7 @@ def run(args): if state.mode == Mode.COMPILE_AND_LINK: # Delay import of link.py to avoid processing this file when only compiling from tools import link + return link.run(linker_inputs, options, state, newargs) else: logger.debug('stopping after compile phase') @@ -749,8 +799,7 @@ def phase_parse_arguments(state): @ToolchainProfiler.profile_block('setup') def phase_setup(options, state, newargs): - """Second phase: configure and setup the compiler based on the specified settings and arguments. - """ + """Second phase: configure and setup the compiler based on the specified settings and arguments.""" if settings.RUNTIME_LINKED_LIBS: newargs += settings.RUNTIME_LINKED_LIBS @@ -775,14 +824,39 @@ def phase_setup(options, state, newargs): continue arg = newargs[i] - if arg in {'-MT', '-MF', '-MJ', '-MQ', '-D', '-U', '-o', '-x', - '-Xpreprocessor', '-include', '-imacros', '-idirafter', - '-iprefix', '-iwithprefix', '-iwithprefixbefore', - '-isysroot', '-imultilib', '-A', '-isystem', '-iquote', - '-install_name', '-compatibility_version', - '-current_version', '-I', '-L', '-include-pch', - '-undefined', '-target', - '-Xlinker', '-Xclang', '-z'}: + if arg in { + '-MT', + '-MF', + '-MJ', + '-MQ', + '-D', + '-U', + '-o', + '-x', + '-Xpreprocessor', + '-include', + '-imacros', + '-idirafter', + '-iprefix', + '-iwithprefix', + '-iwithprefixbefore', + '-isysroot', + '-imultilib', + '-A', + '-isystem', + '-iquote', + '-install_name', + '-compatibility_version', + '-current_version', + '-I', + '-L', + '-include-pch', + '-undefined', + '-target', + '-Xlinker', + '-Xclang', + '-z', + }: skip = True if not arg.startswith('-'): @@ -792,7 +866,11 @@ def phase_setup(options, state, newargs): # python before 3.8: # https://bugs.python.org/issue1311 if not os.path.exists(arg) and arg != os.devnull: - exit_with_error('%s: No such file or directory ("%s" was expected to be an input file, based on the commandline arguments provided)', arg, arg) + exit_with_error( + '%s: No such file or directory ("%s" was expected to be an input file, based on the commandline arguments provided)', + arg, + arg, + ) file_suffix = get_file_suffix(arg) if file_suffix in HEADER_ENDINGS: has_header_inputs = True @@ -843,14 +921,10 @@ def phase_setup(options, state, newargs): if state.mode in (Mode.COMPILE_ONLY, Mode.PREPROCESS_ONLY): for key in user_settings: if key not in COMPILE_TIME_SETTINGS: - diagnostics.warning( - 'unused-command-line-argument', - "linker setting ignored during compilation: '%s'" % key) + diagnostics.warning('unused-command-line-argument', "linker setting ignored during compilation: '%s'" % key) for arg in state.orig_args: if arg in LINK_ONLY_FLAGS: - diagnostics.warning( - 'unused-command-line-argument', - "linker flag ignored during compilation: '%s'" % arg) + diagnostics.warning('unused-command-line-argument', "linker flag ignored during compilation: '%s'" % arg) if settings.MAIN_MODULE or settings.SIDE_MODULE: settings.RELOCATABLE = 1 @@ -870,7 +944,9 @@ def phase_setup(options, state, newargs): # on the command line. This is no longer valid so report either an error or a warning (for # backwards compat with the old `DISABLE_EXCEPTION_CATCHING=2` if user_settings['DISABLE_EXCEPTION_CATCHING'] in ('0', '2'): - diagnostics.warning('deprecated', 'DISABLE_EXCEPTION_CATCHING=X is no longer needed when specifying EXCEPTION_CATCHING_ALLOWED') + diagnostics.warning( + 'deprecated', 'DISABLE_EXCEPTION_CATCHING=X is no longer needed when specifying EXCEPTION_CATCHING_ALLOWED' + ) else: exit_with_error('DISABLE_EXCEPTION_CATCHING and EXCEPTION_CATCHING_ALLOWED are mutually exclusive') @@ -885,18 +961,26 @@ def phase_setup(options, state, newargs): # -fwasm-exceptions takes care of enabling them, so users aren't supposed to # pass them explicitly, regardless of their values if 'DISABLE_EXCEPTION_CATCHING' in user_settings or 'DISABLE_EXCEPTION_THROWING' in user_settings: - diagnostics.warning('emcc', 'you no longer need to pass DISABLE_EXCEPTION_CATCHING or DISABLE_EXCEPTION_THROWING when using Wasm exceptions') + diagnostics.warning( + 'emcc', + 'you no longer need to pass DISABLE_EXCEPTION_CATCHING or DISABLE_EXCEPTION_THROWING when using Wasm exceptions', + ) settings.DISABLE_EXCEPTION_CATCHING = 1 settings.DISABLE_EXCEPTION_THROWING = 1 if user_settings.get('ASYNCIFY') == '1': - diagnostics.warning('emcc', 'ASYNCIFY=1 is not compatible with -fwasm-exceptions. Parts of the program that mix ASYNCIFY and exceptions will not compile.') + diagnostics.warning( + 'emcc', + 'ASYNCIFY=1 is not compatible with -fwasm-exceptions. Parts of the program that mix ASYNCIFY and exceptions will not compile.', + ) if user_settings.get('SUPPORT_LONGJMP') == 'emscripten': exit_with_error('SUPPORT_LONGJMP=emscripten is not compatible with -fwasm-exceptions') if settings.DISABLE_EXCEPTION_THROWING and not settings.DISABLE_EXCEPTION_CATCHING: - exit_with_error("DISABLE_EXCEPTION_THROWING was set (probably from -fno-exceptions) but is not compatible with enabling exception catching (DISABLE_EXCEPTION_CATCHING=0). If you don't want exceptions, set DISABLE_EXCEPTION_CATCHING to 1; if you do want exceptions, don't link with -fno-exceptions") + exit_with_error( + "DISABLE_EXCEPTION_THROWING was set (probably from -fno-exceptions) but is not compatible with enabling exception catching (DISABLE_EXCEPTION_CATCHING=0). If you don't want exceptions, set DISABLE_EXCEPTION_CATCHING to 1; if you do want exceptions, don't link with -fno-exceptions" + ) if options.target.startswith('wasm64'): default_setting('MEMORY64', 1) @@ -1042,7 +1126,10 @@ def get_clang_command_asm(): if options.output_file: cmd += ['-o', options.output_file] if get_file_suffix(options.output_file) == '.bc' and not settings.LTO and '-emit-llvm' not in state.orig_args: - diagnostics.warning('emcc', '.bc output file suffix used without -flto or -emit-llvm. Consider using .o extension since emcc will output an object file, not a bitcode file') + diagnostics.warning( + 'emcc', + '.bc output file suffix used without -flto or -emit-llvm. Consider using .o extension since emcc will output an object file, not a bitcode file', + ) shared.exec_process(cmd) assert False, 'exec_process does not return' @@ -1119,8 +1206,8 @@ def version_string(): revision_suffix = '' if os.path.exists(utils.path_from_root('.git')): git_rev = run_process( - ['git', 'rev-parse', 'HEAD'], - stdout=PIPE, stderr=PIPE, cwd=utils.path_from_root()).stdout.strip() + ['git', 'rev-parse', 'HEAD'], stdout=PIPE, stderr=PIPE, cwd=utils.path_from_root() + ).stdout.strip() revision_suffix = ' (%s)' % git_rev elif os.path.exists(utils.path_from_root('emscripten-revision.txt')): rev = read_file(utils.path_from_root('emscripten-revision.txt')).strip() @@ -1341,7 +1428,9 @@ def consume_arg_file(): elif check_flag('--use-preload-cache'): options.use_preload_cache = True elif check_flag('--no-heap-copy'): - diagnostics.warning('legacy-settings', 'ignoring legacy flag --no-heap-copy (that is the only mode supported now)') + diagnostics.warning( + 'legacy-settings', 'ignoring legacy flag --no-heap-copy (that is the only mode supported now)' + ) elif check_flag('--use-preload-plugins'): options.use_preload_plugins = True elif check_flag('--ignore-dynamic-linking'): @@ -1364,7 +1453,10 @@ def consume_arg_file(): elif check_arg('--js-library'): settings.JS_LIBRARIES.append((i + 1, os.path.abspath(consume_arg_file()))) elif check_flag('--remove-duplicates'): - diagnostics.warning('legacy-settings', '--remove-duplicates is deprecated as it is no longer needed. If you cannot link without it, file a bug with a testcase') + diagnostics.warning( + 'legacy-settings', + '--remove-duplicates is deprecated as it is no longer needed. If you cannot link without it, file a bug with a testcase', + ) elif check_flag('--jcache'): logger.error('jcache is no longer supported') elif check_arg('--cache'): @@ -1376,13 +1468,13 @@ def consume_arg_file(): elif check_flag('--clear-cache'): logger.info('clearing cache as requested by --clear-cache: `%s`', cache.cachedir) cache.erase() - shared.perform_sanity_checks() # this is a good time for a sanity check + shared.perform_sanity_checks() # this is a good time for a sanity check should_exit = True elif check_flag('--clear-ports'): logger.info('clearing ports and cache as requested by --clear-ports') ports.clear() cache.erase() - shared.perform_sanity_checks() # this is a good time for a sanity check + shared.perform_sanity_checks() # this is a good time for a sanity check should_exit = True elif check_flag('--check'): print(version_string(), file=sys.stderr) @@ -1407,10 +1499,12 @@ def consume_arg_file(): # that are e.g. x86 specific and non-portable. The emscripten bundled # headers are modified to be portable, local system ones are generally not. diagnostics.warning( - 'absolute-paths', f'-I or -L of an absolute path "{arg}" ' - 'encountered. If this is to a local system header/library, it may ' - 'cause problems (local system files make sense for compiling natively ' - 'on your system, but not necessarily to JavaScript).') + 'absolute-paths', + f'-I or -L of an absolute path "{arg}" ' + 'encountered. If this is to a local system header/library, it may ' + 'cause problems (local system files make sense for compiling natively ' + 'on your system, but not necessarily to JavaScript).', + ) elif check_flag('--emrun'): options.emrun = True elif check_flag('--cpuprofiler'): @@ -1423,16 +1517,12 @@ def consume_arg_file(): settings.WASM_EXCEPTIONS = 0 elif arg == '-mbulk-memory': settings.BULK_MEMORY = 1 - feature_matrix.enable_feature(feature_matrix.Feature.BULK_MEMORY, - '-mbulk-memory', - override=True) + feature_matrix.enable_feature(feature_matrix.Feature.BULK_MEMORY, '-mbulk-memory', override=True) elif arg == '-mno-bulk-memory': settings.BULK_MEMORY = 0 feature_matrix.disable_feature(feature_matrix.Feature.BULK_MEMORY) elif arg == '-msign-ext': - feature_matrix.enable_feature(feature_matrix.Feature.SIGN_EXT, - '-msign-ext', - override=True) + feature_matrix.enable_feature(feature_matrix.Feature.SIGN_EXT, '-msign-ext', override=True) elif arg == '-mno-sign-ext': feature_matrix.disable_feature(feature_matrix.Feature.SIGN_EXT) elif arg == '-fexceptions': @@ -1482,7 +1572,9 @@ def consume_arg_file(): else: value = '1' if key in settings.keys(): - exit_with_error(f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s{key}={value} instead!') + exit_with_error( + f'{arg}: cannot change built-in settings values with a -jsD directive. Pass -s{key}={value} instead!' + ) user_js_defines += [(key, value)] newargs[i] = '' elif check_flag('-shared'): @@ -1497,7 +1589,9 @@ def consume_arg_file(): elif check_arg('-target') or check_arg('--target'): options.target = consume_arg() if options.target not in ('wasm32', 'wasm64', 'wasm64-unknown-emscripten', 'wasm32-unknown-emscripten'): - exit_with_error(f'unsupported target: {options.target} (emcc only supports wasm64-unknown-emscripten and wasm32-unknown-emscripten)') + exit_with_error( + f'unsupported target: {options.target} (emcc only supports wasm64-unknown-emscripten and wasm32-unknown-emscripten)' + ) elif check_arg('--use-port'): ports.handle_use_port_arg(settings, consume_arg()) elif arg == '-mllvm': @@ -1551,7 +1645,9 @@ def parse_string_value(text): if first == "'" or first == '"': text = text.rstrip() if text[-1] != text[0] or len(text) < 2: - raise ValueError(f'unclosed quoted string. expected final character to be "{text[0]}" and length to be greater than 1 in "{text[0]}"') + raise ValueError( + f'unclosed quoted string. expected final character to be "{text[0]}" and length to be greater than 1 in "{text[0]}"' + ) return text[1:-1] return text @@ -1561,7 +1657,7 @@ def parse_string_list_members(text): result = [] index = 0 while True: - current = values[index].lstrip() # Cannot safely rstrip for cases like: "HERE-> ," + current = values[index].lstrip() # Cannot safely rstrip for cases like: "HERE-> ," if not len(current): raise ValueError('empty value in string list') first = current[0] @@ -1569,7 +1665,7 @@ def parse_string_list_members(text): result.append(current.rstrip()) else: start = index - while True: # Continue until closing quote found + while True: # Continue until closing quote found if index >= len(values): raise ValueError(f"unclosed quoted string. expected final character to be '{first}' in '{values[start]}'") new = values[index].rstrip() @@ -1639,7 +1735,9 @@ def validate_arg_level(level_string, max_level, err_msg, clamp=False): exit_with_error(err_msg) if clamp: if level > max_level: - logger.warning("optimization level '-O" + level_string + "' is not supported; using '-O" + str(max_level) + "' instead") + logger.warning( + "optimization level '-O" + level_string + "' is not supported; using '-O" + str(max_level) + "' instead" + ) level = max_level if not 0 <= level <= max_level: exit_with_error(err_msg) diff --git a/emcmake.py b/emcmake.py index a9dc2294b91c2..84b6c57c424db 100755 --- a/emcmake.py +++ b/emcmake.py @@ -18,12 +18,15 @@ # def run(): if len(sys.argv) < 2 or sys.argv[1] in ('--version', '--help'): - print('''\ + print( + '''\ emcmake is a helper for cmake, setting various environment variables so that emcc etc. are used. Typical usage: emcmake cmake [FLAGS] -''', file=sys.stderr) +''', + file=sys.stderr, + ) return 1 args = sys.argv[1:] @@ -53,7 +56,10 @@ def has_substr(args, substr): elif shutil.which('ninja'): args += ['-G', 'Ninja'] else: - print('emcmake: no compatible cmake generator found; Please install ninja or mingw32-make, or specify a generator explicitly using -G', file=sys.stderr) + print( + 'emcmake: no compatible cmake generator found; Please install ninja or mingw32-make, or specify a generator explicitly using -G', + file=sys.stderr, + ) return 1 print('configure: ' + shared.shlex_join(args), file=sys.stderr) diff --git a/emconfigure.py b/emconfigure.py index deba798069bb3..82d096983c9cf 100755 --- a/emconfigure.py +++ b/emconfigure.py @@ -27,13 +27,16 @@ # def run(): if len(sys.argv) < 2 or sys.argv[1] in ('--version', '--help'): - print('''\ + print( + '''\ emconfigure is a helper for configure, setting various environment variables so that emcc etc. are used. Typical usage: emconfigure ./configure [FLAGS] -(but you can run any command instead of configure)''', file=sys.stderr) +(but you can run any command instead of configure)''', + file=sys.stderr, + ) return 1 args = sys.argv[1:] diff --git a/emmake.py b/emmake.py index 426e5cc4cd63e..62cb5a2f4321d 100755 --- a/emmake.py +++ b/emmake.py @@ -34,13 +34,16 @@ # def run(): if len(sys.argv) < 2 or sys.argv[1] in ('--version', '--help'): - print('''\ + print( + '''\ emmake is a helper for make, setting various environment variables so that emcc etc. are used. Typical usage: emmake make [FLAGS] -(but you can run any command instead of make)''', file=sys.stderr) +(but you can run any command instead of make)''', + file=sys.stderr, + ) return 1 args = sys.argv[1:] diff --git a/emrun.py b/emrun.py index 0d33218c2d747..1c1c8e6606ce1 100644 --- a/emrun.py +++ b/emrun.py @@ -136,8 +136,7 @@ def tick(): def logi(msg): - """Prints a log message to 'info' stdout channel. Always printed. - """ + """Prints a log message to 'info' stdout channel. Always printed.""" global last_message_time with http_mutex: sys.stdout.write(msg + '\n') @@ -158,8 +157,7 @@ def logv(msg): def loge(msg): - """Prints an error message to stderr channel. - """ + """Prints an error message to stderr channel.""" global last_message_time with http_mutex: sys.stderr.write(msg + '\n') @@ -174,8 +172,7 @@ def format_eol(msg): def browser_logi(msg): - """Prints a message to the browser stdout output stream. - """ + """Prints a message to the browser stdout output stream.""" global last_message_time msg = format_eol(msg) browser_stdout_handle.write(msg + '\n') @@ -184,8 +181,7 @@ def browser_logi(msg): def browser_loge(msg): - """Prints a message to the browser stderr output stream. - """ + """Prints a message to the browser stderr output stream.""" global last_message_time msg = format_eol(msg) browser_stderr_handle.write(msg + '\n') @@ -330,13 +326,16 @@ def is_browser_process_alive(): if current_browser_processes: try: import psutil + for p in current_browser_processes: if psutil.pid_exists(p['pid']): return True return False except Exception: # Fail gracefully if psutil not available - logv('psutil is not available, emrun may not be able to accurately track whether the browser process is alive or not') + logv( + 'psutil is not available, emrun may not be able to accurately track whether the browser process is alive or not' + ) # We do not have a track of the browser process ID that we spawned. # Make an assumption that the browser process is open as long until @@ -381,8 +380,12 @@ def kill_browser_process(): else: logv("Terminating all processes that have string '" + processname_killed_atexit + "' in their name.") if WINDOWS: - process_image = processname_killed_atexit if '.exe' in processname_killed_atexit else (processname_killed_atexit + '.exe') - process = subprocess.Popen(['taskkill', '/F', '/IM', process_image, '/T'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process_image = ( + processname_killed_atexit if '.exe' in processname_killed_atexit else (processname_killed_atexit + '.exe') + ) + process = subprocess.Popen( + ['taskkill', '/F', '/IM', process_image, '/T'], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) process.communicate() else: try: @@ -391,7 +394,9 @@ def kill_browser_process(): try: subprocess.call(['killall', processname_killed_atexit]) except OSError: - loge('Both commands pkill and killall failed to clean up the spawned browser process. Perhaps neither of these utilities is available on your system?') + loge( + 'Both commands pkill and killall failed to clean up the spawned browser process. Perhaps neither of these utilities is available on your system?' + ) delete_emrun_safe_firefox_profile() # Clear the process name to represent that the browser is now dead. processname_killed_atexit = '' @@ -408,7 +413,7 @@ def kill_browser_process(): # process that immediately exits. def detect_browser_processes(): if not browser_exe: - return # Running with --no-browser, we are not binding to a spawned browser. + return # Running with --no-browser, we are not binding to a spawned browser. global current_browser_processes logv('First navigation occurred. Identifying currently running browser processes') @@ -421,12 +426,19 @@ def pid_existed(pid): return False for p in running_browser_processes: - logv('Detected running browser process id: ' + str(p['pid']) + ', existed already at emrun startup? ' + str(pid_existed(p['pid']))) + logv( + 'Detected running browser process id: ' + + str(p['pid']) + + ', existed already at emrun startup? ' + + str(pid_existed(p['pid'])) + ) current_browser_processes = [p for p in running_browser_processes if not pid_existed(p['pid'])] if len(current_browser_processes) == 0: - logv('Was unable to detect the browser process that was spawned by emrun. This may occur if the target page was opened in a tab on a browser process that already existed before emrun started up.') + logv( + 'Was unable to detect the browser process that was spawned by emrun. This may occur if the target page was opened in a tab on a browser process that already existed before emrun started up.' + ) # Our custom HTTP web server that will server the target page to run via .html. @@ -437,6 +449,7 @@ def pid_existed(pid): class HTTPWebServer(socketserver.ThreadingMixIn, HTTPServer): """Log messaging arriving via HTTP can come in out of sequence. Implement a sequencing mechanism to enforce ordered transmission.""" + expected_http_seq_num = 1 # Stores messages that have arrived out of order, pending for a send as soon # as the missing message arrives. Kept in sorted order, first element is the @@ -526,7 +539,13 @@ def serve_forever(self, timeout=0.5): time_since_message = now - last_message_time if emrun_options.silence_timeout != 0 and time_since_message > emrun_options.silence_timeout: self.shutdown() - logi('No activity in ' + str(emrun_options.silence_timeout) + ' seconds. Quitting web server with return code ' + str(emrun_options.timeout_returncode) + '. (--silence-timeout option)') + logi( + 'No activity in ' + + str(emrun_options.silence_timeout) + + ' seconds. Quitting web server with return code ' + + str(emrun_options.timeout_returncode) + + '. (--silence-timeout option)' + ) page_exit_code = emrun_options.timeout_returncode emrun_options.kill_exit = True @@ -534,7 +553,13 @@ def serve_forever(self, timeout=0.5): time_since_start = now - page_start_time if emrun_options.timeout != 0 and time_since_start > emrun_options.timeout: self.shutdown() - logi('Page has not finished in ' + str(emrun_options.timeout) + ' seconds. Quitting web server with return code ' + str(emrun_options.timeout_returncode) + '. (--timeout option)') + logi( + 'Page has not finished in ' + + str(emrun_options.timeout) + + ' seconds. Quitting web server with return code ' + + str(emrun_options.timeout_returncode) + + '. (--timeout option)' + ) emrun_options.kill_exit = True page_exit_code = emrun_options.timeout_returncode @@ -542,7 +567,9 @@ def serve_forever(self, timeout=0.5): if not emrun_not_enabled_nag_printed and page_last_served_time is not None: time_since_page_serve = now - page_last_served_time if not have_received_messages and time_since_page_serve > 10: - logv('The html page you are running is not emrun-capable. Stdout, stderr and exit(returncode) capture will not work. Recompile the application with the --emrun linker flag to enable this, or pass --no-emrun-detect to emrun to hide this check.') + logv( + 'The html page you are running is not emrun-capable. Stdout, stderr and exit(returncode) capture will not work. Recompile the application with the --emrun linker flag to enable this, or pass --no-emrun-detect to emrun to hide this check.' + ) emrun_not_enabled_nag_printed = True # Clean up at quit, print any leftover messages in queue. @@ -666,7 +693,7 @@ def do_POST(self): # Binary file dump/upload handling. Requests to # "stdio.html?file=filename" will write binary data to the given file. data = self.rfile.read(int(self.headers['Content-Length'])) - filename = unquote_u(query[len('file='):]) + filename = unquote_u(query[len('file=') :]) filename = os.path.join(emrun_options.dump_out_directory, os.path.normpath(filename)) try: os.makedirs(os.path.dirname(filename)) @@ -698,12 +725,16 @@ def do_POST(self): data = data.replace("+", " ") data = unquote_u(data) - if data == '^pageload^': # Browser is just notifying that it has successfully launched the page. + if data == '^pageload^': # Browser is just notifying that it has successfully launched the page. have_received_messages = True elif data.startswith('^exit^'): if not emrun_options.serve_after_exit: page_exit_code = int(data[6:]) - logv('Web page has quit with a call to exit() with return code ' + str(page_exit_code) + '. Shutting down web server. Pass --serve-after-exit to keep serving even after the page terminates with exit().') + logv( + 'Web page has quit with a call to exit() with return code ' + + str(page_exit_code) + + '. Shutting down web server. Pass --serve-after-exit to keep serving even after the page terminates with exit().' + ) # Set server socket to nonblocking on shutdown to avoid sporadic deadlocks self.server.socket.setblocking(False) self.server.shutdown() @@ -722,7 +753,7 @@ def do_POST(self): try: i = data.index('^', 5) seq_num = int(data[5:i]) - data = data[i + 1:] + data = data[i + 1 :] except ValueError: pass @@ -741,7 +772,9 @@ def do_POST(self): # Returns stdout by running command with universal_newlines=True def check_output(cmd, universal_newlines=True, *args, **kwargs): if hasattr(subprocess, "run"): - return subprocess.run(cmd, universal_newlines=universal_newlines, stdout=subprocess.PIPE, check=True, *args, **kwargs).stdout + return subprocess.run( + cmd, universal_newlines=universal_newlines, stdout=subprocess.PIPE, check=True, *args, **kwargs + ).stdout else: # check_output is considered as an old API so prefer subprocess.run if possible return subprocess.check_output(cmd, universal_newlines=universal_newlines, *args, **kwargs) @@ -757,6 +790,7 @@ def get_cpu_info(): try: if WINDOWS: from win32com.client import GetObject + root_winmgmts = GetObject('winmgmts:root\\cimv2') cpus = root_winmgmts.ExecQuery('Select * from Win32_Processor') cpu_name = cpus[0].Name + ', ' + platform.processor() @@ -780,18 +814,16 @@ def get_cpu_info(): logical_cores = physical_cores * int(re.search(r'Thread\(s\) per core: (.*)', lscpu).group(1).strip()) except Exception as e: import traceback + loge(traceback.format_exc()) - return {'model': 'Unknown ("' + str(e) + '")', - 'physicalCores': 1, - 'logicalCores': 1, - 'frequency': 0 - } + return {'model': 'Unknown ("' + str(e) + '")', 'physicalCores': 1, 'logicalCores': 1, 'frequency': 0} - return {'model': platform.machine() + ', ' + cpu_name, - 'physicalCores': physical_cores, - 'logicalCores': logical_cores, - 'frequency': frequency - } + return { + 'model': platform.machine() + ', ' + cpu_name, + 'physicalCores': physical_cores, + 'logicalCores': logical_cores, + 'frequency': frequency, + } def get_android_cpu_infoline(): @@ -800,11 +832,13 @@ def get_android_cpu_infoline(): hardware = '' for line in lines: if line.startswith('Processor'): - processor = line[line.find(':') + 1:].strip() + processor = line[line.find(':') + 1 :].strip() elif line.startswith('Hardware'): - hardware = line[line.find(':') + 1:].strip() + hardware = line[line.find(':') + 1 :].strip() - freq = int(check_output([ADB, 'shell', 'cat', '/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq']).strip()) // 1000 + freq = ( + int(check_output([ADB, 'shell', 'cat', '/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq']).strip()) // 1000 + ) return 'CPU: ' + processor + ', ' + hardware + ' @ ' + str(freq) + ' MHz' @@ -866,11 +900,11 @@ def linux_get_gpu_info(): glxinfo = check_output('glxinfo') for line in glxinfo.split("\n"): if "OpenGL vendor string:" in line: - gl_vendor = line[len("OpenGL vendor string:"):].strip() + gl_vendor = line[len("OpenGL vendor string:") :].strip() if "OpenGL version string:" in line: - gl_version = line[len("OpenGL version string:"):].strip() + gl_version = line[len("OpenGL version string:") :].strip() if "OpenGL renderer string:" in line: - gl_renderer = line[len("OpenGL renderer string:"):].strip() + gl_renderer = line[len("OpenGL renderer string:") :].strip() glinfo = gl_vendor + ' ' + gl_renderer + ', GL version ' + gl_version except Exception as e: logv(e) @@ -929,13 +963,14 @@ def get_executable_version(filename): try: if WINDOWS: import win32api + info = win32api.GetFileVersionInfo(filename, "\\") ms = info['FileVersionMS'] ls = info['FileVersionLS'] version = win32api.HIWORD(ms), win32api.LOWORD(ms), win32api.HIWORD(ls), win32api.LOWORD(ls) return '.'.join(map(str, version)) elif MACOS: - plistfile = filename[0:filename.find('MacOS')] + 'Info.plist' + plistfile = filename[0 : filename.find('MacOS')] + 'Info.plist' info = plistlib.readPlist(plistfile) # Data in Info.plists is a bit odd, this check combo gives best information on each browser. if 'firefox' in filename.lower(): @@ -959,7 +994,7 @@ def get_executable_version(filename): def get_browser_build_date(filename): try: if MACOS: - plistfile = filename[0:filename.find('MacOS')] + 'Info.plist' + plistfile = filename[0 : filename.find('MacOS')] + 'Info.plist' info = plistlib.readPlist(plistfile) # Data in Info.plists is a bit odd, this check combo gives best information on each browser. if 'firefox' in filename.lower(): @@ -979,32 +1014,55 @@ def get_browser_build_date(filename): def get_browser_info(filename, format_json): if format_json: - return json.dumps({ - 'name': browser_display_name(filename), - 'version': get_executable_version(filename), - 'buildDate': get_browser_build_date(filename) - }, indent=2) + return json.dumps( + { + 'name': browser_display_name(filename), + 'version': get_executable_version(filename), + 'buildDate': get_browser_build_date(filename), + }, + indent=2, + ) else: - return 'Browser: ' + browser_display_name(filename) + ' ' + get_executable_version(filename) + ', build ' + get_browser_build_date(filename) + return ( + 'Browser: ' + + browser_display_name(filename) + + ' ' + + get_executable_version(filename) + + ', build ' + + get_browser_build_date(filename) + ) # http://stackoverflow.com/questions/580924/python-windows-file-version-attribute def win_get_file_properties(fname): - propNames = ('Comments', 'InternalName', 'ProductName', - 'CompanyName', 'LegalCopyright', 'ProductVersion', - 'FileDescription', 'LegalTrademarks', 'PrivateBuild', - 'FileVersion', 'OriginalFilename', 'SpecialBuild') + propNames = ( + 'Comments', + 'InternalName', + 'ProductName', + 'CompanyName', + 'LegalCopyright', + 'ProductVersion', + 'FileDescription', + 'LegalTrademarks', + 'PrivateBuild', + 'FileVersion', + 'OriginalFilename', + 'SpecialBuild', + ) props = {'FixedFileInfo': None, 'StringFileInfo': None, 'FileVersion': None} import win32api + # backslash as parm returns dictionary of numeric info corresponding to VS_FIXEDFILEINFO struct fixedInfo = win32api.GetFileVersionInfo(fname, '\\') props['FixedFileInfo'] = fixedInfo - props['FileVersion'] = "%d.%d.%d.%d" % (fixedInfo['FileVersionMS'] / 65536, - fixedInfo['FileVersionMS'] % 65536, - fixedInfo['FileVersionLS'] / 65536, - fixedInfo['FileVersionLS'] % 65536) + props['FileVersion'] = "%d.%d.%d.%d" % ( + fixedInfo['FileVersionMS'] / 65536, + fixedInfo['FileVersionMS'] % 65536, + fixedInfo['FileVersionLS'] / 65536, + fixedInfo['FileVersionLS'] % 65536, + ) # \VarFileInfo\Translation returns list of available (language, codepage) # pairs that can be used to retrieve string info. We are using only the first pair. @@ -1015,7 +1073,7 @@ def win_get_file_properties(fname): strInfo = {} for propName in propNames: - strInfoPath = u'\\StringFileInfo\\%04X%04X\\%s' % (lang, codepage, propName) + strInfoPath = '\\StringFileInfo\\%04X%04X\\%s' % (lang, codepage, propName) ## print str_info strInfo[propName] = win32api.GetFileVersionInfo(fname, strInfoPath) @@ -1045,7 +1103,7 @@ def get_computer_model(): model = re.search('(.*)', model) model = model.group(1).strip() with open(os.path.join(os.getenv("HOME"), '.emrun.hwmodel.cached'), 'w') as fh: - fh.write(model) # Cache the hardware model to disk + fh.write(model) # Cache the hardware model to disk return model except Exception: hwmodel = check_output(['sysctl', 'hw.model']) @@ -1067,7 +1125,20 @@ def get_computer_model(): bios_vendor = check_output(['cat', '/sys/devices/virtual/dmi/id/bios_vendor']).strip() bios_version = check_output(['cat', '/sys/devices/virtual/dmi/id/bios_version']).strip() bios_date = check_output(['cat', '/sys/devices/virtual/dmi/id/bios_date']).strip() - return board_vendor + ' ' + board_name + ' ' + board_version + ', ' + bios_vendor + ' ' + bios_version + ' (' + bios_date + ')' + return ( + board_vendor + + ' ' + + board_name + + ' ' + + board_version + + ', ' + + bios_vendor + + ' ' + + bios_version + + ' (' + + bios_date + + ')' + ) except Exception as e: logv(str(e)) return 'Generic' @@ -1090,7 +1161,14 @@ def get_os_version(): return 'macOS ' + platform.mac_ver()[0] + bitness elif LINUX: kernel_version = check_output(['uname', '-r']).strip() - return ' '.join(platform.linux_distribution()) + ', linux kernel ' + kernel_version + ' ' + platform.architecture()[0] + bitness + return ( + ' '.join(platform.linux_distribution()) + + ', linux kernel ' + + kernel_version + + ' ' + + platform.architecture()[0] + + bitness + ) except Exception: return 'Unknown OS' @@ -1110,6 +1188,7 @@ def get_system_memory(): return int(sline[1]) * 1024 elif WINDOWS: import win32api + return win32api.GlobalMemoryStatusEx()['TotalPhys'] elif MACOS: return int(check_output(['sysctl', '-n', 'hw.memsize']).strip()) @@ -1192,14 +1271,16 @@ def find_browser(name): if MACOS: # Note: by default Firefox beta installs as 'Firefox.app', you must manually rename it to # FirefoxBeta.app after installation. - browser_locations = [('firefox', '/Applications/Firefox.app/Contents/MacOS/firefox'), - ('firefox_beta', '/Applications/FirefoxBeta.app/Contents/MacOS/firefox'), - ('firefox_aurora', '/Applications/FirefoxAurora.app/Contents/MacOS/firefox'), - ('firefox_nightly', '/Applications/FirefoxNightly.app/Contents/MacOS/firefox'), - ('safari', '/Applications/Safari.app/Contents/MacOS/Safari'), - ('opera', '/Applications/Opera.app/Contents/MacOS/Opera'), - ('chrome', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'), - ('chrome_canary', '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary')] + browser_locations = [ + ('firefox', '/Applications/Firefox.app/Contents/MacOS/firefox'), + ('firefox_beta', '/Applications/FirefoxBeta.app/Contents/MacOS/firefox'), + ('firefox_aurora', '/Applications/FirefoxAurora.app/Contents/MacOS/firefox'), + ('firefox_nightly', '/Applications/FirefoxNightly.app/Contents/MacOS/firefox'), + ('safari', '/Applications/Safari.app/Contents/MacOS/Safari'), + ('opera', '/Applications/Opera.app/Contents/MacOS/Opera'), + ('chrome', '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'), + ('chrome_canary', '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary'), + ] elif WINDOWS: pf_locations = ['ProgramFiles(x86)', 'ProgramFiles', 'ProgramW6432', 'LOCALAPPDATA'] @@ -1208,31 +1289,35 @@ def find_browser(name): continue program_files = os.environ[pf_env] if WINDOWS else '' - browser_locations += [('chrome', os.path.join(program_files, 'Google/Chrome/Application/chrome.exe')), - ('chrome_canary', os.path.expanduser("~/AppData/Local/Google/Chrome SxS/Application/chrome.exe")), - ('firefox_nightly', os.path.join(program_files, 'Nightly/firefox.exe')), - ('firefox_aurora', os.path.join(program_files, 'Aurora/firefox.exe')), - ('firefox_beta', os.path.join(program_files, 'Beta/firefox.exe')), - ('firefox_beta', os.path.join(program_files, 'FirefoxBeta/firefox.exe')), - ('firefox_beta', os.path.join(program_files, 'Firefox Beta/firefox.exe')), - ('firefox', os.path.join(program_files, 'Mozilla Firefox/firefox.exe')), - ('iexplore', os.path.join(program_files, 'Internet Explorer/iexplore.exe')), - ('opera', os.path.join(program_files, 'Opera/launcher.exe'))] + browser_locations += [ + ('chrome', os.path.join(program_files, 'Google/Chrome/Application/chrome.exe')), + ('chrome_canary', os.path.expanduser("~/AppData/Local/Google/Chrome SxS/Application/chrome.exe")), + ('firefox_nightly', os.path.join(program_files, 'Nightly/firefox.exe')), + ('firefox_aurora', os.path.join(program_files, 'Aurora/firefox.exe')), + ('firefox_beta', os.path.join(program_files, 'Beta/firefox.exe')), + ('firefox_beta', os.path.join(program_files, 'FirefoxBeta/firefox.exe')), + ('firefox_beta', os.path.join(program_files, 'Firefox Beta/firefox.exe')), + ('firefox', os.path.join(program_files, 'Mozilla Firefox/firefox.exe')), + ('iexplore', os.path.join(program_files, 'Internet Explorer/iexplore.exe')), + ('opera', os.path.join(program_files, 'Opera/launcher.exe')), + ] elif LINUX: - browser_locations = [('firefox', os.path.expanduser('~/firefox/firefox')), - ('firefox_beta', os.path.expanduser('~/firefox_beta/firefox')), - ('firefox_aurora', os.path.expanduser('~/firefox_aurora/firefox')), - ('firefox_nightly', os.path.expanduser('~/firefox_nightly/firefox')), - ('chrome', which('google-chrome-stable')), - ('chrome', which('google-chrome'))] + browser_locations = [ + ('firefox', os.path.expanduser('~/firefox/firefox')), + ('firefox_beta', os.path.expanduser('~/firefox_beta/firefox')), + ('firefox_aurora', os.path.expanduser('~/firefox_aurora/firefox')), + ('firefox_nightly', os.path.expanduser('~/firefox_nightly/firefox')), + ('chrome', which('google-chrome-stable')), + ('chrome', which('google-chrome')), + ] for alias, browser_exe in browser_locations: if name == alias: if browser_exe is not None and os.path.isfile(browser_exe): return [browser_exe] - return None # Could not find the browser + return None # Could not find the browser def get_android_model(): @@ -1295,7 +1380,17 @@ def list_android_browsers(): def list_pc_browsers(): - browsers = ['firefox', 'firefox_beta', 'firefox_aurora', 'firefox_nightly', 'chrome', 'chrome_canary', 'iexplore', 'safari', 'opera'] + browsers = [ + 'firefox', + 'firefox_beta', + 'firefox_aurora', + 'firefox_nightly', + 'chrome', + 'chrome_canary', + 'iexplore', + 'safari', + 'opera', + ] logi('emrun has automatically found the following browsers in the default install locations on the system:') logi('') for browser in browsers: @@ -1306,7 +1401,9 @@ def list_pc_browsers(): logi(' - ' + browser + ': ' + browser_display_name(browser_exe) + ' ' + get_executable_version(browser_exe)) logi('') logi('You can pass the --browser option to launch with the given browser above.') - logi('Even if your browser was not detected, you can use --browser /path/to/browser/executable to launch with that browser.') + logi( + 'Even if your browser was not detected, you can use --browser /path/to/browser/executable to launch with that browser.' + ) def browser_display_name(browser): @@ -1335,8 +1432,8 @@ def subprocess_env(): e = os.environ.copy() # https://bugzilla.mozilla.org/show_bug.cgi?id=745154 e['MOZ_DISABLE_AUTO_SAFE_MODE'] = '1' - e['MOZ_DISABLE_SAFE_MODE_KEY'] = '1' # https://bugzilla.mozilla.org/show_bug.cgi?id=653410#c9 - e['JIT_OPTION_asmJSAtomicsEnable'] = 'true' # https://bugzilla.mozilla.org/show_bug.cgi?id=1299359#c0 + e['MOZ_DISABLE_SAFE_MODE_KEY'] = '1' # https://bugzilla.mozilla.org/show_bug.cgi?id=653410#c9 + e['JIT_OPTION_asmJSAtomicsEnable'] = 'true' # https://bugzilla.mozilla.org/show_bug.cgi?id=1299359#c0 return e @@ -1344,12 +1441,14 @@ def subprocess_env(): def remove_tree(d): os.chmod(d, stat.S_IWRITE) try: + def remove_readonly_and_try_again(func, path, _exc_info): if not (os.stat(path).st_mode & stat.S_IWRITE): os.chmod(path, stat.S_IWRITE) func(path) else: raise + shutil.rmtree(d, onerror=remove_readonly_and_try_again) except Exception: pass @@ -1358,14 +1457,20 @@ def remove_readonly_and_try_again(func, path, _exc_info): def get_system_info(format_json): if emrun_options.android: if format_json: - return json.dumps({'model': get_android_model(), - 'os': get_android_os_version(), - 'ram': get_system_memory(), - 'cpu': get_android_cpu_infoline() - }, indent=2) + return json.dumps( + { + 'model': get_android_model(), + 'os': get_android_os_version(), + 'ram': get_system_memory(), + 'cpu': get_android_cpu_infoline(), + }, + indent=2, + ) else: info = 'Model: ' + get_android_model() + '\n' - info += 'OS: ' + get_android_os_version() + ' with ' + str(get_system_memory() // 1024 // 1024) + ' MB of System RAM\n' + info += ( + 'OS: ' + get_android_os_version() + ' with ' + str(get_system_memory() // 1024 // 1024) + ' MB of System RAM\n' + ) info += 'CPU: ' + get_android_cpu_infoline() + '\n' return info.strip() else: @@ -1374,6 +1479,7 @@ def get_system_info(format_json): unique_system_id = fh.read().strip() except Exception: import uuid + unique_system_id = str(uuid.uuid4()) try: open(os.path.expanduser('~/.emrun.generated.guid'), 'w').write(unique_system_id) @@ -1381,25 +1487,44 @@ def get_system_info(format_json): logv(e) if format_json: - return json.dumps({'name': socket.gethostname(), - 'model': get_computer_model(), - 'os': get_os_version(), - 'ram': get_system_memory(), - 'cpu': get_cpu_info(), - 'gpu': get_gpu_info(), - 'uuid': unique_system_id}, indent=2) + return json.dumps( + { + 'name': socket.gethostname(), + 'model': get_computer_model(), + 'os': get_os_version(), + 'ram': get_system_memory(), + 'cpu': get_cpu_info(), + 'gpu': get_gpu_info(), + 'uuid': unique_system_id, + }, + indent=2, + ) else: cpu = get_cpu_info() gpus = get_gpu_info() - info = 'Computer name: ' + socket.gethostname() + '\n' # http://stackoverflow.com/questions/799767/getting-name-of-windows-computer-running-python-script + info = ( + 'Computer name: ' + socket.gethostname() + '\n' + ) # http://stackoverflow.com/questions/799767/getting-name-of-windows-computer-running-python-script info += 'Model: ' + get_computer_model() + '\n' info += 'OS: ' + get_os_version() + ' with ' + str(get_system_memory() // 1024 // 1024) + ' MB of System RAM\n' - info += 'CPU: ' + cpu['model'] + ', ' + str(cpu['frequency']) + ' MHz, ' + str(cpu['physicalCores']) + ' physical cores, ' + str(cpu['logicalCores']) + ' logical cores\n' + info += ( + 'CPU: ' + + cpu['model'] + + ', ' + + str(cpu['frequency']) + + ' MHz, ' + + str(cpu['physicalCores']) + + ' physical cores, ' + + str(cpu['logicalCores']) + + ' logical cores\n' + ) if len(gpus) == 1: info += 'GPU: ' + gpus[0]['model'] + ' with ' + str(gpus[0]['ram'] // 1024 // 1024) + " MB of VRAM\n" elif len(gpus) > 1: for i in range(0, len(gpus)): - info += 'GPU' + str(i) + ": " + gpus[i]['model'] + ' with ' + str(gpus[i]['ram'] // 1024 // 1024) + ' MBs of VRAM\n' + info += ( + 'GPU' + str(i) + ": " + gpus[i]['model'] + ' with ' + str(gpus[i]['ram'] // 1024 // 1024) + ' MBs of VRAM\n' + ) info += 'UUID: ' + unique_system_id return info.strip() @@ -1416,6 +1541,7 @@ def list_processes_by_name(exe_full_path): pids = [] try: import psutil + for proc in psutil.process_iter(): try: pinfo = proc.as_dict(attrs=['pid', 'name', 'exe']) @@ -1449,121 +1575,170 @@ def list_processes_by_name(exe_full_path): def parse_args(args): parser = argparse.ArgumentParser(usage=usage_str) - parser.add_argument('--kill-start', action='store_true', - help='If true, any previously running instances of ' - 'the target browser are killed before starting.') - - parser.add_argument('--kill-exit', action='store_true', - help='If true, the spawned browser process is forcibly ' - 'killed when it calls exit(). Note: Using this ' - 'option may require explicitly passing the option ' - '--browser=/path/to/browser, to avoid emrun being ' - 'detached from the browser process it spawns.') - - parser.add_argument('--no-server', dest='run_server', action='store_false', - default=True, - help='If specified, a HTTP web server is not launched ' - 'to host the page to run.') - - parser.add_argument('--no-browser', dest='run_browser', action='store_false', - default=True, - help='If specified, emrun will not launch a web browser ' - 'to run the page.') - - parser.add_argument('--no-emrun-detect', action='store_true', - help='If specified, skips printing the warning message ' - 'if html page is detected to not have been built ' - 'with --emrun linker flag.') - - parser.add_argument('--serve-after-close', action='store_true', - help='If true, serves the web page even after the ' - 'application quits by user closing the web page.') - - parser.add_argument('--serve-after-exit', action='store_true', - help='If true, serves the web page even after the ' - 'application quits by a call to exit().') - - parser.add_argument('--serve-root', - help='If set, specifies the root path that the emrun ' - 'web server serves. If not specified, the directory ' - 'where the target .html page lives in is served.') - - parser.add_argument('--verbose', action='store_true', - help='Enable verbose logging from emrun internal operation.') - - parser.add_argument('--hostname', default=default_webserver_hostname, - help='Specifies the hostname the server runs in.') - - parser.add_argument('--port', default=default_webserver_port, type=int, - help='Specifies the port the server runs in.') - - parser.add_argument('--log-stdout', - help='Specifies a log filename where the browser process ' - 'stdout data will be appended to.') - - parser.add_argument('--log-stderr', - help='Specifies a log filename where the browser process stderr data will be appended to.') - - parser.add_argument('--silence-timeout', type=int, default=0, - help='If no activity is received in this many seconds, ' - 'the browser process is assumed to be hung, and the web ' - 'server is shut down and the target browser killed. ' - 'Disabled by default.') - - parser.add_argument('--timeout', type=int, default=0, - help='If the browser process does not quit or the page ' - 'exit() in this many seconds, the browser is assumed ' - 'to be hung, and the web server is shut down and the ' - 'target browser killed. Disabled by default.') - - parser.add_argument('--timeout-returncode', type=int, default=99999, - help='Sets the exit code that emrun reports back to ' - 'caller in the case that a page timeout occurs. ' - 'Default: 99999.') - - parser.add_argument('--list-browsers', action='store_true', - help='Prints out all detected browser that emrun is able ' - 'to use with the --browser command and exits.') - - parser.add_argument('--browser', - help='Specifies the browser executable to run the web page in.') - - parser.add_argument('--browser-args', default='', - help='Specifies the arguments to the browser executable.') - - parser.add_argument('--android', action='store_true', - help='Launches the page in a browser of an Android ' - 'device connected to an USB on the local system. (via adb)') - - parser.add_argument('--android-tunnel', action='store_true', - help='Expose the port directly to the Android device ' - 'and connect to it as localhost, establishing ' - 'cross origin isolation. Implies --android. A ' - 'reverse socket connection is created by adb ' - 'reverse, and remains after emrun terminates (it ' - 'can be removed by adb reverse --remove).') - - parser.add_argument('--system-info', action='store_true', - help='Prints information about the current system at startup.') - - parser.add_argument('--browser-info', action='store_true', - help='Prints information about the target browser to launch at startup.') - - parser.add_argument('--json', action='store_true', - help='If specified, --system-info and --browser-info are ' - 'outputted in JSON format.') - - parser.add_argument('--safe-firefox-profile', action='store_true', - help='If true, the browser is launched into a new clean ' - 'Firefox profile that is suitable for unattended ' - 'automated runs. (If target browser != Firefox, ' - 'this parameter is ignored)') - - parser.add_argument('--private-browsing', action='store_true', - help='If specified, opens browser in private/incognito mode.') - - parser.add_argument('--dump-out-directory', default='dump_out', type=str, - help='If specified, overrides the directory for dump files using emrun_file_dump method.') + parser.add_argument( + '--kill-start', + action='store_true', + help='If true, any previously running instances of ' 'the target browser are killed before starting.', + ) + + parser.add_argument( + '--kill-exit', + action='store_true', + help='If true, the spawned browser process is forcibly ' + 'killed when it calls exit(). Note: Using this ' + 'option may require explicitly passing the option ' + '--browser=/path/to/browser, to avoid emrun being ' + 'detached from the browser process it spawns.', + ) + + parser.add_argument( + '--no-server', + dest='run_server', + action='store_false', + default=True, + help='If specified, a HTTP web server is not launched ' 'to host the page to run.', + ) + + parser.add_argument( + '--no-browser', + dest='run_browser', + action='store_false', + default=True, + help='If specified, emrun will not launch a web browser ' 'to run the page.', + ) + + parser.add_argument( + '--no-emrun-detect', + action='store_true', + help='If specified, skips printing the warning message ' + 'if html page is detected to not have been built ' + 'with --emrun linker flag.', + ) + + parser.add_argument( + '--serve-after-close', + action='store_true', + help='If true, serves the web page even after the ' 'application quits by user closing the web page.', + ) + + parser.add_argument( + '--serve-after-exit', + action='store_true', + help='If true, serves the web page even after the ' 'application quits by a call to exit().', + ) + + parser.add_argument( + '--serve-root', + help='If set, specifies the root path that the emrun ' + 'web server serves. If not specified, the directory ' + 'where the target .html page lives in is served.', + ) + + parser.add_argument('--verbose', action='store_true', help='Enable verbose logging from emrun internal operation.') + + parser.add_argument( + '--hostname', default=default_webserver_hostname, help='Specifies the hostname the server runs in.' + ) + + parser.add_argument('--port', default=default_webserver_port, type=int, help='Specifies the port the server runs in.') + + parser.add_argument( + '--log-stdout', help='Specifies a log filename where the browser process ' 'stdout data will be appended to.' + ) + + parser.add_argument( + '--log-stderr', help='Specifies a log filename where the browser process stderr data will be appended to.' + ) + + parser.add_argument( + '--silence-timeout', + type=int, + default=0, + help='If no activity is received in this many seconds, ' + 'the browser process is assumed to be hung, and the web ' + 'server is shut down and the target browser killed. ' + 'Disabled by default.', + ) + + parser.add_argument( + '--timeout', + type=int, + default=0, + help='If the browser process does not quit or the page ' + 'exit() in this many seconds, the browser is assumed ' + 'to be hung, and the web server is shut down and the ' + 'target browser killed. Disabled by default.', + ) + + parser.add_argument( + '--timeout-returncode', + type=int, + default=99999, + help='Sets the exit code that emrun reports back to ' + 'caller in the case that a page timeout occurs. ' + 'Default: 99999.', + ) + + parser.add_argument( + '--list-browsers', + action='store_true', + help='Prints out all detected browser that emrun is able ' 'to use with the --browser command and exits.', + ) + + parser.add_argument('--browser', help='Specifies the browser executable to run the web page in.') + + parser.add_argument('--browser-args', default='', help='Specifies the arguments to the browser executable.') + + parser.add_argument( + '--android', + action='store_true', + help='Launches the page in a browser of an Android ' 'device connected to an USB on the local system. (via adb)', + ) + + parser.add_argument( + '--android-tunnel', + action='store_true', + help='Expose the port directly to the Android device ' + 'and connect to it as localhost, establishing ' + 'cross origin isolation. Implies --android. A ' + 'reverse socket connection is created by adb ' + 'reverse, and remains after emrun terminates (it ' + 'can be removed by adb reverse --remove).', + ) + + parser.add_argument( + '--system-info', action='store_true', help='Prints information about the current system at startup.' + ) + + parser.add_argument( + '--browser-info', action='store_true', help='Prints information about the target browser to launch at startup.' + ) + + parser.add_argument( + '--json', + action='store_true', + help='If specified, --system-info and --browser-info are ' 'outputted in JSON format.', + ) + + parser.add_argument( + '--safe-firefox-profile', + action='store_true', + help='If true, the browser is launched into a new clean ' + 'Firefox profile that is suitable for unattended ' + 'automated runs. (If target browser != Firefox, ' + 'this parameter is ignored)', + ) + + parser.add_argument( + '--private-browsing', action='store_true', help='If specified, opens browser in private/incognito mode.' + ) + + parser.add_argument( + '--dump-out-directory', + default='dump_out', + type=str, + help='If specified, overrides the directory for dump files using emrun_file_dump method.', + ) parser.add_argument('serve', nargs='?', default='') @@ -1637,7 +1812,9 @@ def run(args): # noqa: C901, PLR0912, PLR0915 file_to_serve = options.serve else: file_to_serve = '.' - file_to_serve_is_url = file_to_serve.startswith('file://') or file_to_serve.startswith('http://') or file_to_serve.startswith('https://') + file_to_serve_is_url = ( + file_to_serve.startswith('file://') or file_to_serve.startswith('http://') or file_to_serve.startswith('https://') + ) if options.serve_root: serve_dir = os.path.abspath(options.serve_root) @@ -1680,7 +1857,9 @@ def run(args): # noqa: C901, PLR0912, PLR0915 if options.android: if options.run_browser or options.browser_info: if not options.browser: - loge("Running on Android requires that you explicitly specify the browser to run with --browser . Run emrun --android --list-browsers to obtain a list of installed browsers you can use.") + loge( + "Running on Android requires that you explicitly specify the browser to run with --browser . Run emrun --android --list-browsers to obtain a list of installed browsers you can use." + ) return 1 elif options.browser == 'firefox': browser_app = 'org.mozilla.firefox/org.mozilla.gecko.BrowserApp' @@ -1713,15 +1892,19 @@ def run(args): # noqa: C901, PLR0912, PLR0915 url = url.replace('&', '\\&') browser = [ADB, 'shell', 'am', 'start', '-a', 'android.intent.action.VIEW', '-n', browser_app, '-d', url] - processname_killed_atexit = browser_app[:browser_app.find('/')] - else: # Launching a web page on local system. + processname_killed_atexit = browser_app[: browser_app.find('/')] + else: # Launching a web page on local system. if options.browser: options.browser = unwrap(options.browser) if options.run_browser or options.browser_info: browser = find_browser(str(options.browser)) if not browser: - loge('Unable to find browser "' + str(options.browser) + '"! Check the correctness of the passed --browser=xxx parameter!') + loge( + 'Unable to find browser "' + + str(options.browser) + + '"! Check the correctness of the passed --browser=xxx parameter!' + ) return 1 browser_exe = browser[0] browser_args = shlex.split(unwrap(options.browser_args)) @@ -1737,12 +1920,20 @@ def run(args): # noqa: C901, PLR0912, PLR0915 processname_killed_atexit = 'Safari' elif 'chrome' in browser_exe.lower(): processname_killed_atexit = 'chrome' - browser_args += ['--enable-nacl', '--enable-pnacl', '--disable-restore-session-state', '--enable-webgl', - '--no-default-browser-check', '--no-first-run', '--allow-file-access-from-files', '--password-store=basic'] + browser_args += [ + '--enable-nacl', + '--enable-pnacl', + '--disable-restore-session-state', + '--enable-webgl', + '--no-default-browser-check', + '--no-first-run', + '--allow-file-access-from-files', + '--password-store=basic', + ] if options.private_browsing: browser_args += ['--incognito'] - # if not options.run_server: - # browser_args += ['--disable-web-security'] + # if not options.run_server: + # browser_args += ['--disable-web-security'] elif 'firefox' in browser_exe.lower(): processname_killed_atexit = 'firefox' elif 'iexplore' in browser_exe.lower(): @@ -1777,7 +1968,9 @@ def run(cmd): run(['adb', 'shell', 'mkdir', '/mnt/sdcard/safe_firefox_profile']) run(['adb', 'push', os.path.join(profile_dir, 'prefs.js'), '/mnt/sdcard/safe_firefox_profile/prefs.js']) except Exception as e: - loge('Creating Firefox profile prefs.js file to internal storage in /mnt/sdcard failed with error ' + str(e) + '!') + loge( + 'Creating Firefox profile prefs.js file to internal storage in /mnt/sdcard failed with error ' + str(e) + '!' + ) loge('Try running without --safe-firefox-profile flag if unattended execution mode is not important, or') loge('enable rooted debugging on the Android device to allow adb to write files to /mnt/sdcard.') browser += ['--es', 'args', '"--profile /mnt/sdcard/safe_firefox_profile"'] @@ -1785,7 +1978,12 @@ def run(cmd): # Create temporary Firefox profile to run the page with. This is important to # run after kill_browser_process()/kill_start op above, since that cleans up # the temporary profile if one exists. - if processname_killed_atexit == 'firefox' and options.safe_firefox_profile and options.run_browser and not options.android: + if ( + processname_killed_atexit == 'firefox' + and options.safe_firefox_profile + and options.run_browser + and not options.android + ): profile_dir = create_emrun_safe_firefox_profile() browser += ['-no-remote', '--profile', profile_dir.replace('\\', '/')] @@ -1827,7 +2025,12 @@ def run(cmd): logv(browser_exe) previous_browser_processes = list_processes_by_name(browser_exe) for p in previous_browser_processes: - logv('Before spawning web browser, found a running ' + os.path.basename(browser_exe) + ' browser process id: ' + str(p['pid'])) + logv( + 'Before spawning web browser, found a running ' + + os.path.basename(browser_exe) + + ' browser process id: ' + + str(p['pid']) + ) browser_process = subprocess.Popen(browser, env=subprocess_env()) logv('Launched browser process with pid=' + str(browser_process.pid)) if options.kill_exit: @@ -1841,9 +2044,15 @@ def run(cmd): premature_quit_code = browser_process.poll() if premature_quit_code is not None: options.serve_after_close = True - logv('Warning: emrun got immediately detached from the target browser process (the process quit with exit code ' + str(premature_quit_code) + '). Cannot detect when user closes the browser. Behaving as if --serve-after-close was passed in.') + logv( + 'Warning: emrun got immediately detached from the target browser process (the process quit with exit code ' + + str(premature_quit_code) + + '). Cannot detect when user closes the browser. Behaving as if --serve-after-close was passed in.' + ) if not options.browser: - logv('Try passing the --browser=/path/to/browser option to avoid this from occurring. See https://github.com/emscripten-core/emscripten/issues/3234 for more discussion.') + logv( + 'Try passing the --browser=/path/to/browser option to avoid this from occurring. See https://github.com/emscripten-core/emscripten/issues/3234 for more discussion.' + ) if options.run_server: try: @@ -1880,7 +2089,11 @@ def main(args): returncode = run(args) logv('emrun quitting with process exit code ' + str(returncode)) if temp_firefox_profile_dir is not None: - logi('Warning: Had to leave behind a temporary Firefox profile directory ' + temp_firefox_profile_dir + ' because --safe-firefox-profile was set and the browser did not quit before emrun did.') + logi( + 'Warning: Had to leave behind a temporary Firefox profile directory ' + + temp_firefox_profile_dir + + ' because --safe-firefox-profile was set and the browser did not quit before emrun did.' + ) return returncode diff --git a/emsize.py b/emsize.py index 8bf92a3843c01..6ea849351f20a 100755 --- a/emsize.py +++ b/emsize.py @@ -61,8 +61,7 @@ def print_sizes(js_file): if not os.path.isfile(wasm_file): return error('Wasm file %s not found' % wasm_file) - sizes = shared.check_call([LLVM_SIZE, '--format=sysv', wasm_file], - stdout=subprocess.PIPE).stdout + sizes = shared.check_call([LLVM_SIZE, '--format=sysv', wasm_file], stdout=subprocess.PIPE).stdout # llvm-size may emit some number of blank lines (after the total), ignore them lines = [line for line in sizes.splitlines() if line] diff --git a/emstrip.py b/emstrip.py index f054fe7e9f1df..b330234f960cb 100755 --- a/emstrip.py +++ b/emstrip.py @@ -4,8 +4,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Wrapper script around `llvm-strip`. -""" +"""Wrapper script around `llvm-strip`.""" import sys from tools import shared diff --git a/emsymbolizer.py b/emsymbolizer.py index 7ba3b951c852b..37728882b53fb 100755 --- a/emsymbolizer.py +++ b/emsymbolizer.py @@ -22,8 +22,7 @@ from tools import shared from tools import webassembly -LLVM_SYMBOLIZER = os.path.expanduser( - shared.build_llvm_tool_path(shared.exe_suffix('llvm-symbolizer'))) +LLVM_SYMBOLIZER = os.path.expanduser(shared.build_llvm_tool_path(shared.exe_suffix('llvm-symbolizer'))) class Error(BaseException): @@ -68,8 +67,7 @@ def symbolize_address_symbolizer(module, address, is_dwarf): vma_adjust = get_codesec_offset(module) else: vma_adjust = 0 - cmd = [LLVM_SYMBOLIZER, '-e', module.filename, f'--adjust-vma={vma_adjust}', - str(address)] + cmd = [LLVM_SYMBOLIZER, '-e', module.filename, f'--adjust-vma={vma_adjust}', str(address)] out = shared.run_process(cmd, stdout=subprocess.PIPE).stdout.strip() out_lines = out.splitlines() @@ -184,11 +182,7 @@ def lookup(self, offset): nearest = self.find_offset(offset) assert nearest in self.mappings, 'Sourcemap has an offset with no mapping' info = self.mappings[nearest] - return LocationInfo( - self.sources[info.source] if info.source is not None else None, - info.line, - info.column - ) + return LocationInfo(self.sources[info.source] if info.source is not None else None, info.line, info.column) def symbolize_address_sourcemap(module, address, force_file): @@ -223,36 +217,32 @@ def main(args): if args.addrtype == 'code': address += get_codesec_offset(module) - if ((has_debug_line_section(module) and not args.source) or - 'dwarf' in args.source): + if (has_debug_line_section(module) and not args.source) or 'dwarf' in args.source: symbolize_address_symbolizer(module, address, is_dwarf=True) - elif ((get_sourceMappingURL_section(module) and not args.source) or - 'sourcemap' in args.source): + elif (get_sourceMappingURL_section(module) and not args.source) or 'sourcemap' in args.source: symbolize_address_sourcemap(module, address, args.file) - elif ((has_name_section(module) and not args.source) or - 'names' in args.source): + elif (has_name_section(module) and not args.source) or 'names' in args.source: symbolize_address_symbolizer(module, address, is_dwarf=False) - elif ((has_linking_section(module) and not args.source) or - 'symtab' in args.source): + elif (has_linking_section(module) and not args.source) or 'symtab' in args.source: symbolize_address_symbolizer(module, address, is_dwarf=False) else: - raise Error('No .debug_line or sourceMappingURL section found in ' - f'{module.filename}.' - " I don't know how to symbolize this file yet") + raise Error( + 'No .debug_line or sourceMappingURL section found in ' + f'{module.filename}.' + " I don't know how to symbolize this file yet" + ) def get_args(): parser = argparse.ArgumentParser() - parser.add_argument('-s', '--source', choices=['dwarf', 'sourcemap', - 'names', 'symtab'], - help='Force debug info source type', default=()) - parser.add_argument('-f', '--file', action='store', - help='Force debug info source file') - parser.add_argument('-t', '--addrtype', choices=['code', 'file'], - default='file', - help='Address type (code section or file offset)') - parser.add_argument('-v', '--verbose', action='store_true', - help='Print verbose info for debugging this script') + parser.add_argument( + '-s', '--source', choices=['dwarf', 'sourcemap', 'names', 'symtab'], help='Force debug info source type', default=() + ) + parser.add_argument('-f', '--file', action='store', help='Force debug info source file') + parser.add_argument( + '-t', '--addrtype', choices=['code', 'file'], default='file', help='Address type (code section or file offset)' + ) + parser.add_argument('-v', '--verbose', action='store_true', help='Print verbose info for debugging this script') parser.add_argument('wasm_file', help='Wasm file') parser.add_argument('address', help='Address to lookup') args = parser.parse_args() diff --git a/pyproject.toml b/pyproject.toml index 874462532f8e3..825e5e34451ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,12 @@ exclude = [ "./tools/scons/", ".git", ] +format.exclude = [ + "./test/test_browser.py", +] +format.quote-style = "preserve" +indent-width = 2 +line-length = 120 lint.select = [ "ARG", diff --git a/requirements-dev.txt b/requirements-dev.txt index 84c824c5ee1b6..97d17eaf370d5 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ coverage[toml]==5.5 mypy==0.971 -ruff==0.8.2 +ruff==0.8.3 types-requests==2.27.14 unittest-xml-reporting==3.1.0 diff --git a/site/source/get_api_items.py b/site/source/get_api_items.py index ee94f6909e6d3..404bb42eee026 100755 --- a/site/source/get_api_items.py +++ b/site/source/get_api_items.py @@ -26,77 +26,83 @@ def parseFiles(): - """Parse api-reference files to extract the code items. - """ - - def addapiitems(matchobj): - # print 'matcobj0: %s' % matchobj.group(0) - # print 'matcobj1: %s' % matchobj.group(1) - # print 'matcobj2: %s' % matchobj.group(2) - # print 'matcobj3: %s' % matchobj.group(3) - # print 'matcobj4: %s' % matchobj.group(4) - - lang = matchobj.group(2) - data_type = matchobj.group(3) - if data_type == 'function': - data_type = 'func' - api_item = matchobj.group(4) - api_item = api_item.strip() - api_item = api_item.split('(')[0] - try: - api_item = api_item.split(' ')[1] - except IndexError: - pass - - # print lang - # print data_type - # print api_item - - api_reference_items[api_item] = ':%s:%s:`%s`' % (lang, data_type, api_item) - # Add additional index for functions declared as func() rather than just func - if data_type == 'func': - api_item_index = api_item + '()' - api_reference_items[api_item_index] = ':%s:%s:`%s`' % (lang, data_type, api_item) - - # print api_reference_items[api_item] - - for file in os.listdir(api_reference_directory): - if file.endswith(".rst"): - filepath = api_reference_directory + file - print(file) - # open file - with open(filepath, 'r') as infile: - for line in infile: - # parse line for API items - re.sub(r'^\.\.\s+((\w+)\:(\w+)\:\:(.*))', addapiitems, line) + """Parse api-reference files to extract the code items.""" + + def addapiitems(matchobj): + # print 'matcobj0: %s' % matchobj.group(0) + # print 'matcobj1: %s' % matchobj.group(1) + # print 'matcobj2: %s' % matchobj.group(2) + # print 'matcobj3: %s' % matchobj.group(3) + # print 'matcobj4: %s' % matchobj.group(4) + + lang = matchobj.group(2) + data_type = matchobj.group(3) + if data_type == 'function': + data_type = 'func' + api_item = matchobj.group(4) + api_item = api_item.strip() + api_item = api_item.split('(')[0] + try: + api_item = api_item.split(' ')[1] + except IndexError: + pass + + # print lang + # print data_type + # print api_item + + api_reference_items[api_item] = ':%s:%s:`%s`' % (lang, data_type, api_item) + # Add additional index for functions declared as func() rather than just func + if data_type == 'func': + api_item_index = api_item + '()' + api_reference_items[api_item_index] = ':%s:%s:`%s`' % (lang, data_type, api_item) + + # print api_reference_items[api_item] + + for file in os.listdir(api_reference_directory): + if file.endswith(".rst"): + filepath = api_reference_directory + file + print(file) + # open file + with open(filepath, 'r') as infile: + for line in infile: + # parse line for API items + re.sub(r'^\.\.\s+((\w+)\:(\w+)\:\:(.*))', addapiitems, line) def exportItems(): - """Export the API items into form for use in another script. - """ - with open(api_item_filename, 'w') as infile: - # write function lead in - infile.write("# Auto-generated file (see get_api_items.py)\n\ndef get_mapped_items():\n mapped_wiki_inline_code = dict()\n") + """Export the API items into form for use in another script.""" + with open(api_item_filename, 'w') as infile: + # write function lead in + infile.write( + "# Auto-generated file (see get_api_items.py)\n\ndef get_mapped_items():\n mapped_wiki_inline_code = dict()\n" + ) - items = list((key, value) for key, value in api_reference_items.items()) - items.sort() - for key, value in items: - # Write out each API item to add - infile.write(" mapped_wiki_inline_code['%s'] = '%s'\n" % (key, value)) + items = list((key, value) for key, value in api_reference_items.items()) + items.sort() + for key, value in items: + # Write out each API item to add + infile.write(" mapped_wiki_inline_code['%s'] = '%s'\n" % (key, value)) - # write the return function - infile.write(" return mapped_wiki_inline_code\n") + # write the return function + infile.write(" return mapped_wiki_inline_code\n") def main(): - parser = optparse.OptionParser(usage="Usage: %prog [options] version") - parser.add_option("-s", "--siteapi", dest="siteapi", default="http://www.developer.nokia.com/Community/Wiki/api.php", help="Location of API") - (options, args) = parser.parse_args() - # print 'Site: %s' % options.siteapi - parseFiles() - exportItems() - return 0 + parser = optparse.OptionParser(usage="Usage: %prog [options] version") + parser.add_option( + "-s", + "--siteapi", + dest="siteapi", + default="http://www.developer.nokia.com/Community/Wiki/api.php", + help="Location of API", + ) + (options, args) = parser.parse_args() + # print 'Site: %s' % options.siteapi + parseFiles() + exportItems() + return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/site/source/get_wiki.py b/site/source/get_wiki.py index 899531541f222..dbe2a3c5e5b78 100755 --- a/site/source/get_wiki.py +++ b/site/source/get_wiki.py @@ -39,185 +39,197 @@ temp_set_of_codemarkup = set() logfile = open(logfilename, 'w') # snapshot_version_information = '.. note:: This is a **snapshot** of the wiki: %s\n\n' % strftime("%a, %d %b %Y %H:%M", gmtime()) -snapshot_version_information = '.. note:: This article was migrated from the wiki (%s) and is now the "master copy" (the version in the wiki will be deleted). It may not be a perfect rendering of the original but we hope to fix that soon!\n\n' % time.strftime("%a, %d %b %Y %H:%M", time.gmtime()) +snapshot_version_information = ( + '.. note:: This article was migrated from the wiki (%s) and is now the "master copy" (the version in the wiki will be deleted). It may not be a perfect rendering of the original but we hope to fix that soon!\n\n' + % time.strftime("%a, %d %b %Y %H:%M", time.gmtime()) +) def CleanWiki(): - """Delete the wiki clone directory and all contained files. - """ + """Delete the wiki clone directory and all contained files.""" - def errorhandler(func, path, exc_info): - # where func is os.listdir, os.remove, or os.rmdir; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info() - print(func) - print(path) - print(exc_info) - os.chmod(path, stat.S_IWRITE) - os.remove(path) + def errorhandler(func, path, exc_info): + # where func is os.listdir, os.remove, or os.rmdir; path is the argument to that function that caused it to fail; and exc_info is a tuple returned by sys.exc_info() + print(func) + print(path) + print(exc_info) + os.chmod(path, stat.S_IWRITE) + os.remove(path) - try: - shutil.rmtree(output_dir, ignore_errors=False, onerror=errorhandler) - print('Old wiki clone removed') - except IOError: - print('No directory to clean found') + try: + shutil.rmtree(output_dir, ignore_errors=False, onerror=errorhandler) + print('Old wiki clone removed') + except IOError: + print('No directory to clean found') def CloneWiki(): - """ - Clone the wiki into a temporary location (first cleaning) - """ - # Clean up existing repo - CleanWiki() + """ + Clone the wiki into a temporary location (first cleaning) + """ + # Clean up existing repo + CleanWiki() - # Create directory for output and temporary files - try: - os.makedirs(output_dir) - print('Created directory') - except OSError: - pass + # Create directory for output and temporary files + try: + os.makedirs(output_dir) + print('Created directory') + except OSError: + pass - # Clone - git_clone_command = 'git clone %s %s' % (wiki_repo, wiki_checkout) - print(git_clone_command) - os.system(git_clone_command) + # Clone + git_clone_command = 'git clone %s %s' % (wiki_repo, wiki_checkout) + print(git_clone_command) + os.system(git_clone_command) def ConvertFilesToRst(): - """ - Add template to specified page object (wikitools) - """ - indexfiletext = '============================\nWiki snapshot (ready-for-review)\n============================\n\n%s\n.. toctree::\n :maxdepth: 2\n' % snapshot_version_information - for file in os.listdir(wiki_checkout): - if not file.endswith(".md"): - continue - - inputfilename = wiki_checkout + file - markdown = Path(inputfilename).read_text() - if 'This article has moved from the wiki to the new site' in markdown: - continue - if 'This page has been migrated to the main site' in markdown: - continue - - print(file) - # get name of file - filenamestripped = os.path.splitext(file)[0] - indexfiletext += '\n %s' % filenamestripped - outputfilename = output_dir + filenamestripped + '.rst' - - command = 'pandoc -f markdown -t rst -o "%s" "%s"' % (outputfilename, inputfilename) - print(command) - if os.system(command): - sys.exit(1) - title = filenamestripped.replace('-', ' ') - # print title - logfile.write('title from filename: %s \n' % title) - # add import message to title - title += ' (wiki-import)' - length = len(title) - # print length - headerbar = '' - for _ in range(length): - headerbar += '=' - page_reference = filenamestripped - page_reference_link_text = '.. _%s:\n\n' % page_reference - titlebar = page_reference_link_text + headerbar + '\n' + title + '\n' + headerbar + '\n' - textinfile = '' - # Add titlebar to start of the file (from the filename) - textinfile += titlebar - # Add wiki snapshot information - - textinfile += snapshot_version_information - - with open(outputfilename) as infile: - for line in infile: - textinfile += line - - # print textinfile - with open(outputfilename, 'w') as outfile: - outfile.write(textinfile) - - # write the index - with open(output_dir + 'index.rst', 'w') as outfile: - outfile.write(indexfiletext) + """ + Add template to specified page object (wikitools) + """ + indexfiletext = ( + '============================\nWiki snapshot (ready-for-review)\n============================\n\n%s\n.. toctree::\n :maxdepth: 2\n' + % snapshot_version_information + ) + for file in os.listdir(wiki_checkout): + if not file.endswith(".md"): + continue + + inputfilename = wiki_checkout + file + markdown = Path(inputfilename).read_text() + if 'This article has moved from the wiki to the new site' in markdown: + continue + if 'This page has been migrated to the main site' in markdown: + continue + + print(file) + # get name of file + filenamestripped = os.path.splitext(file)[0] + indexfiletext += '\n %s' % filenamestripped + outputfilename = output_dir + filenamestripped + '.rst' + + command = 'pandoc -f markdown -t rst -o "%s" "%s"' % (outputfilename, inputfilename) + print(command) + if os.system(command): + sys.exit(1) + title = filenamestripped.replace('-', ' ') + # print title + logfile.write('title from filename: %s \n' % title) + # add import message to title + title += ' (wiki-import)' + length = len(title) + # print length + headerbar = '' + for _ in range(length): + headerbar += '=' + page_reference = filenamestripped + page_reference_link_text = '.. _%s:\n\n' % page_reference + titlebar = page_reference_link_text + headerbar + '\n' + title + '\n' + headerbar + '\n' + textinfile = '' + # Add titlebar to start of the file (from the filename) + textinfile += titlebar + # Add wiki snapshot information + + textinfile += snapshot_version_information + + with open(outputfilename) as infile: + for line in infile: + textinfile += line + + # print textinfile + with open(outputfilename, 'w') as outfile: + outfile.write(textinfile) + + # write the index + with open(output_dir + 'index.rst', 'w') as outfile: + outfile.write(indexfiletext) def FixupConvertedRstFiles(): - """Add template to specified page object (wikitools) + """Add template to specified page object (wikitools)""" + + def fixInternalWikiLinks(aOldText): + """ + Fixes wiki links in [[linkname]] format by changing this to a document link in current directory. + """ + + def fixwikilinks(matchobj): + # print 'matcobj0: %s' % matchobj.group(0) + # print 'matcobj1: %s' % matchobj.group(1) + linktext = matchobj.group(1) + linktext = linktext.replace(' ', '-') + # linktext = ':doc:`%s`' % linktext + # use reference for linking as allows pages to be moved around + linktext = ':ref:`%s`' % linktext + # print 'linkdoc: %s' % linktext + logfile.write('linkdoc: %s \n' % linktext) + return linktext + + # print 'fixing wiki links' + return re.sub(r'\[\[(.+?)\]\]', fixwikilinks, aOldText) + + def fixWikiCodeMarkupToCodeLinks(aOldText): """ + Links "known" code objects if they are found in wiki markup. + """ + + def fixcodemarkuplinks(matchobj): + # print 'Inline code: %s' % matchobj.group(0) + # print 'matcobj1: %s' % matchobj.group(1) + temp_set_of_codemarkup.add(matchobj.group(0)) + linktext = matchobj.group(1) + if linktext in mapped_wiki_inline_code: + logfile.write('Replace: %s \n' % mapped_wiki_inline_code[linktext]) + return mapped_wiki_inline_code[linktext] + + return matchobj.group(0) # linktext + + # print 'fixing up code markup to code reference' + return re.sub(r'``(.+?)``', fixcodemarkuplinks, aOldText) + + for file in os.listdir(output_dir): + if file.endswith(".rst"): + input_file = output_dir + file + # print input_file + textinfile = '' + with open(input_file) as infile: + for line in infile: + textinfile += line + + # print textinfile + # fix up broken wiki-page links in files + textinfile = fixInternalWikiLinks(textinfile) + + # convert codemarkup to links if possible + textinfile = fixWikiCodeMarkupToCodeLinks(textinfile) + + with open(input_file, 'w') as outfile: + outfile.write(textinfile) - def fixInternalWikiLinks(aOldText): - """ - Fixes wiki links in [[linkname]] format by changing this to a document link in current directory. - """ - def fixwikilinks(matchobj): - # print 'matcobj0: %s' % matchobj.group(0) - # print 'matcobj1: %s' % matchobj.group(1) - linktext = matchobj.group(1) - linktext = linktext.replace(' ', '-') - # linktext = ':doc:`%s`' % linktext - # use reference for linking as allows pages to be moved around - linktext = ':ref:`%s`' % linktext - # print 'linkdoc: %s' % linktext - logfile.write('linkdoc: %s \n' % linktext) - return linktext - # print 'fixing wiki links' - return re.sub(r'\[\[(.+?)\]\]', fixwikilinks, aOldText) - - def fixWikiCodeMarkupToCodeLinks(aOldText): - """ - Links "known" code objects if they are found in wiki markup. - """ - def fixcodemarkuplinks(matchobj): - # print 'Inline code: %s' % matchobj.group(0) - # print 'matcobj1: %s' % matchobj.group(1) - temp_set_of_codemarkup.add(matchobj.group(0)) - linktext = matchobj.group(1) - if linktext in mapped_wiki_inline_code: - logfile.write('Replace: %s \n' % mapped_wiki_inline_code[linktext]) - return mapped_wiki_inline_code[linktext] - - return matchobj.group(0) # linktext - # print 'fixing up code markup to code reference' - return re.sub(r'``(.+?)``', fixcodemarkuplinks, aOldText) - - for file in os.listdir(output_dir): - if file.endswith(".rst"): - input_file = output_dir + file - # print input_file - textinfile = '' - with open(input_file) as infile: - for line in infile: - textinfile += line - - # print textinfile - # fix up broken wiki-page links in files - textinfile = fixInternalWikiLinks(textinfile) - - # convert codemarkup to links if possible - textinfile = fixWikiCodeMarkupToCodeLinks(textinfile) - - with open(input_file, 'w') as outfile: - outfile.write(textinfile) - - logfile.write('\n\nCODE MARKUP THAT WONT BE LINKED (add entry to mapped_wiki_inline_code if one of these need to be linked. The tool get-api-items.py can be used to generate the list of the documented API items. \n') - for item in temp_set_of_codemarkup: - logfile.write('%s\n' % item) + logfile.write( + '\n\nCODE MARKUP THAT WONT BE LINKED (add entry to mapped_wiki_inline_code if one of these need to be linked. The tool get-api-items.py can be used to generate the list of the documented API items. \n' + ) + for item in temp_set_of_codemarkup: + logfile.write('%s\n' % item) # parser options def main(): - parser = optparse.OptionParser(version="%prog 0.1.1", usage="Usage: %prog [options] version") - parser.add_option("-c", "--clonewiki", action="store_true", default=False, dest="clonewiki", help="Clean and clone the latest wiki") - options, args = parser.parse_args() + parser = optparse.OptionParser(version="%prog 0.1.1", usage="Usage: %prog [options] version") + parser.add_option( + "-c", "--clonewiki", action="store_true", default=False, dest="clonewiki", help="Clean and clone the latest wiki" + ) + options, args = parser.parse_args() - print('Clone wiki: %s' % options.clonewiki) - if options.clonewiki: - CloneWiki() - # input = raw_input('CHECK ALL files were cloned! (look for "error: unable to create file" )\n') + print('Clone wiki: %s' % options.clonewiki) + if options.clonewiki: + CloneWiki() + # input = raw_input('CHECK ALL files were cloned! (look for "error: unable to create file" )\n') - ConvertFilesToRst() - FixupConvertedRstFiles() - print('See LOG for details: %s ' % logfilename) + ConvertFilesToRst() + FixupConvertedRstFiles() + print('See LOG for details: %s ' % logfilename) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) diff --git a/system/bin/sdl-config.py b/system/bin/sdl-config.py index e028a4f7397be..9d1bb92a72622 100755 --- a/system/bin/sdl-config.py +++ b/system/bin/sdl-config.py @@ -11,4 +11,3 @@ print('') elif '--version' in args: print('1.3.0') - diff --git a/system/bin/sdl2-config.py b/system/bin/sdl2-config.py index 957ef5900a3ec..bb8266c7dabec 100755 --- a/system/bin/sdl2-config.py +++ b/system/bin/sdl2-config.py @@ -10,4 +10,3 @@ print('-sUSE_SDL=2') elif '--version' in args: print('2.0.10') - diff --git a/test/benchmark/benchmark_sse.py b/test/benchmark/benchmark_sse.py index 65f296c1dafce..e24f60e2e208a 100644 --- a/test/benchmark/benchmark_sse.py +++ b/test/benchmark/benchmark_sse.py @@ -34,161 +34,195 @@ def run_benchmark(benchmark_file, results_file, build_args): - # Run native build - out_file = os.path.join(temp_dir, 'benchmark_sse_native') - if WINDOWS: - out_file += '.exe' - cmd = [CLANG_CXX] + clang_native.get_clang_native_args() + [benchmark_file, '-O3', '-o', out_file] - print('Building native version of the benchmark:') - print(' '.join(cmd)) - run_process(cmd, env=clang_native.get_clang_native_env()) + # Run native build + out_file = os.path.join(temp_dir, 'benchmark_sse_native') + if WINDOWS: + out_file += '.exe' + cmd = [CLANG_CXX] + clang_native.get_clang_native_args() + [benchmark_file, '-O3', '-o', out_file] + print('Building native version of the benchmark:') + print(' '.join(cmd)) + run_process(cmd, env=clang_native.get_clang_native_env()) - native_results = Popen([out_file], stdout=PIPE, stderr=PIPE).communicate() - print(native_results[0]) + native_results = Popen([out_file], stdout=PIPE, stderr=PIPE).communicate() + print(native_results[0]) - # Run emscripten build - out_file = os.path.join(temp_dir, 'benchmark_sse_html.js') - cmd = [EMCC, benchmark_file, '-O3', '-sTOTAL_MEMORY=536870912', '-o', out_file] + build_args - print('Building Emscripten version of the benchmark:') - print(' '.join(cmd)) - run_process(cmd) + # Run emscripten build + out_file = os.path.join(temp_dir, 'benchmark_sse_html.js') + cmd = [EMCC, benchmark_file, '-O3', '-sTOTAL_MEMORY=536870912', '-o', out_file] + build_args + print('Building Emscripten version of the benchmark:') + print(' '.join(cmd)) + run_process(cmd) - cmd = V8_ENGINE + ['--experimental-wasm-simd', os.path.basename(out_file)] - print(' '.join(cmd)) - old_dir = os.getcwd() - os.chdir(os.path.dirname(out_file)) - wasm_results = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() - os.chdir(old_dir) + cmd = V8_ENGINE + ['--experimental-wasm-simd', os.path.basename(out_file)] + print(' '.join(cmd)) + old_dir = os.getcwd() + os.chdir(os.path.dirname(out_file)) + wasm_results = Popen(cmd, stdout=PIPE, stderr=PIPE).communicate() + os.chdir(old_dir) - if not wasm_results: - raise Exception('Unable to run benchmark in V8!') + if not wasm_results: + raise Exception('Unable to run benchmark in V8!') - if not wasm_results[0].strip(): - print(wasm_results[1]) - sys.exit(1) + if not wasm_results[0].strip(): + print(wasm_results[1]) + sys.exit(1) - print(wasm_results[0]) + print(wasm_results[0]) - def strip_comments(text): - return re.sub('//.*?\n|/\*.*?\*/', '', text, re.S) # noqa + def strip_comments(text): + return re.sub('//.*?\n|/\*.*?\*/', '', text, re.S) # noqa - benchmark_results = strip_comments(wasm_results[0]) + benchmark_results = strip_comments(wasm_results[0]) - # Strip out unwanted print output. - benchmark_results = benchmark_results[benchmark_results.find('{'):].strip() - if '*************************' in benchmark_results: - benchmark_results = benchmark_results[:benchmark_results.find('*************************')].strip() + # Strip out unwanted print output. + benchmark_results = benchmark_results[benchmark_results.find('{') :].strip() + if '*************************' in benchmark_results: + benchmark_results = benchmark_results[: benchmark_results.find('*************************')].strip() - print(benchmark_results) + print(benchmark_results) - shutil.rmtree(temp_dir) + shutil.rmtree(temp_dir) - native_results = json.loads(native_results[0]) - benchmark_results = benchmark_results[benchmark_results.index('{'):benchmark_results.rindex('}') + 1] - wasm_results = json.loads(benchmark_results) + native_results = json.loads(native_results[0]) + benchmark_results = benchmark_results[benchmark_results.index('{') : benchmark_results.rindex('}') + 1] + wasm_results = json.loads(benchmark_results) - # native_workload = native_results['workload'] - # html_workload = wasm_results['workload'] + # native_workload = native_results['workload'] + # html_workload = wasm_results['workload'] - html = '''

SSE JavaScript Benchmark

+ html = ( + '''

SSE JavaScript Benchmark

System Info:
- ''' + system_info[0].replace('\n', '
') + ''' + ''' + + system_info[0].replace('\n', '
') + + ''' Native Clang Compiler:
- ''' + native_info[1].replace('\n', '
') + ''' + ''' + + native_info[1].replace('\n', '
') + + ''' Emscripten Compiler:
- ''' + emscripten_info[0].replace('\n', '
') - - charts_native = {} - charts_html = {} - for result in native_results['results']: - ch = result['chart'] - if ch not in charts_native: - charts_native[ch] = [] - charts_native[ch] += [result] - for result in wasm_results['results']: - ch = result['chart'] - if ch not in charts_html: - charts_html[ch] = [] - charts_html[ch] += [result] - - def find_result_in_category(results, category): - for result in results: - if result['category'] == category: - return result - return None - - def format_comparison(a, b): - if a < b and a != 0: - return " {:10.2f}".format(b / a) + 'x FASTER' - elif b != 0: - return " {:10.2f}".format(a / b) + 'x SLOWER' - else: - return " NaN " - - chartNumber = 0 - - total_time_native_scalar = 0 - total_time_native_simd = 0 - total_time_html_scalar = 0 - total_time_html_simd = 0 - - for chart_name, chart_native_results in charts_native.items(): - # Extract data for each chart. - categories = [] - nativeScalarResults = [] - nativeSimdResults = [] - htmlScalarResults = [] - htmlSimdResults = [] - native_results = chart_native_results - wasm_results = charts_html[chart_name] - textual_results_native = '

' - textual_results_html = '

' - textual_results_html2 = '

' - textual_results_html3 = '

' - for result in native_results: - categories += ["'" + result['category'] + "'"] - nsc = result['scalar'] - nsi = result['simd'] - nativeScalarResults += [str(nsc)] - nativeSimdResults += [str(nsi)] - html_result = find_result_in_category(wasm_results, result['category']) - textual_results_native += 'Native ' + result['category'] + ': ' + "{:10.4f}".format(nsc) + 'ns -> ' + "{:10.4f}".format(nsi) + 'ns. ' - textual_results_native += 'Native SSE is ' + format_comparison(nsi, nsc) + ' than native scalar.        
' - - if html_result is not None: - hsc = html_result['scalar'] - htmlScalarResults += [str(hsc)] - hsi = html_result['simd'] - htmlSimdResults += [str(hsi)] - textual_results_html += 'JS ' + result['category'] + ': ' + "{:10.4f}".format(hsc) + 'ns -> ' + "{:10.4f}".format(hsi) + 'ns. ' - textual_results_html += 'JS SSE is ' + format_comparison(hsi, hsc) + ' than JS scalar.        
' - textual_results_html2 += 'JS ' + result['category'] + ': JS scalar is ' + format_comparison(hsc, nsc) + ' than native scalar.        
' - textual_results_html3 += 'JS ' + result['category'] + ': JS SSE is ' + format_comparison(hsi, nsi) + ' than native SSE.        
' - total_time_native_scalar += nsc - total_time_native_simd += nsi - total_time_html_scalar += hsc - total_time_html_simd += hsi - else: - htmlScalarResults += [str(-1)] - htmlSimdResults += [str(-1)] - - chartNumber += 1 - html += '

' - html += '''''' + '
' + textual_results_native + '' + textual_results_html + '
' + textual_results_html2 + '' + textual_results_html3 + '
' - - # Final overall score - - html += '
' - html += '''''' + + '
' + + textual_results_native + + '' + + textual_results_html + + '
' + + textual_results_html2 + + '' + + textual_results_html3 + + '
' + ) + + # Final overall score + + html += '
' + html += ( + '''''' + ) - html += '' + html += '' - open(results_file, 'w').write(html) - print('Wrote ' + str(len(html)) + ' bytes to file ' + results_file + '.') + open(results_file, 'w').write(html) + print('Wrote ' + str(len(html)) + ' bytes to file ' + results_file + '.') if __name__ == '__main__': - suite = sys.argv[1].lower() if len(sys.argv) == 2 else None - if suite in ['sse', 'sse1']: - run_benchmark(test_file('sse/benchmark_sse1.cpp'), 'results_sse1.html', ['-msse']) - elif suite == 'sse2': - run_benchmark(test_file('sse/benchmark_sse2.cpp'), 'results_sse2.html', ['-msse2']) - elif suite == 'sse3': - run_benchmark(test_file('sse/benchmark_sse3.cpp'), 'results_sse3.html', ['-msse3']) - elif suite == 'ssse3': - run_benchmark(test_file('sse/benchmark_ssse3.cpp'), 'results_ssse3.html', ['-mssse3']) - else: - raise Exception('Usage: python test/benchmark_sse.py sse1|sse2|sse3') + suite = sys.argv[1].lower() if len(sys.argv) == 2 else None + if suite in ['sse', 'sse1']: + run_benchmark(test_file('sse/benchmark_sse1.cpp'), 'results_sse1.html', ['-msse']) + elif suite == 'sse2': + run_benchmark(test_file('sse/benchmark_sse2.cpp'), 'results_sse2.html', ['-msse2']) + elif suite == 'sse3': + run_benchmark(test_file('sse/benchmark_sse3.cpp'), 'results_sse3.html', ['-msse3']) + elif suite == 'ssse3': + run_benchmark(test_file('sse/benchmark_ssse3.cpp'), 'results_ssse3.html', ['-mssse3']) + else: + raise Exception('Usage: python test/benchmark_sse.py sse1|sse2|sse3') diff --git a/test/clang_native.py b/test/clang_native.py index 1193eb969bf52..027bd5205f2f5 100644 --- a/test/clang_native.py +++ b/test/clang_native.py @@ -15,15 +15,15 @@ def get_native_triple(): arch = { - 'aarch64': 'arm64', - 'arm64': 'arm64', - 'x86_64': 'x86_64', - 'AMD64': 'x86_64', + 'aarch64': 'arm64', + 'arm64': 'arm64', + 'x86_64': 'x86_64', + 'AMD64': 'x86_64', }[platform.machine()] OS = { - 'linux': 'linux', - 'darwin': 'darwin', - 'win32': 'windows-msvc', + 'linux': 'linux', + 'darwin': 'darwin', + 'win32': 'windows-msvc', }[sys.platform] return f'{arch}-{OS}' @@ -78,7 +78,11 @@ def get_clang_native_env(): else: visual_studio_path = 'C:\\Program Files (x86)\\Microsoft Visual Studio 14.0' if not os.path.isdir(visual_studio_path): - raise Exception('Visual Studio 2015 was not found in "' + visual_studio_path + '"! Run in Visual Studio X64 command prompt to avoid the need to autoguess this location (or set VSINSTALLDIR env var).') + raise Exception( + 'Visual Studio 2015 was not found in "' + + visual_studio_path + + '"! Run in Visual Studio X64 command prompt to avoid the need to autoguess this location (or set VSINSTALLDIR env var).' + ) # Guess where Program Files (x86) is located if 'ProgramFiles(x86)' in env: @@ -98,19 +102,31 @@ def get_clang_native_env(): else: windows8_sdk_dir = os.path.join(prog_files_x86, 'Windows Kits', '8.1') if not os.path.isdir(windows8_sdk_dir): - raise Exception('Windows 8.1 SDK was not found in "' + windows8_sdk_dir + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location (or set WindowsSdkDir env var).') + raise Exception( + 'Windows 8.1 SDK was not found in "' + + windows8_sdk_dir + + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location (or set WindowsSdkDir env var).' + ) # Guess where Windows 10 SDK is located if os.path.isdir(os.path.join(prog_files_x86, 'Windows Kits', '10')): windows10_sdk_dir = os.path.join(prog_files_x86, 'Windows Kits', '10') if not os.path.isdir(windows10_sdk_dir): - raise Exception('Windows 10 SDK was not found in "' + windows10_sdk_dir + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location.') + raise Exception( + 'Windows 10 SDK was not found in "' + + windows10_sdk_dir + + '"! Run in Visual Studio command prompt to avoid the need to autoguess this location.' + ) env.setdefault('VSINSTALLDIR', visual_studio_path) env.setdefault('VCINSTALLDIR', os.path.join(visual_studio_path, 'VC')) windows10sdk_kits_include_dir = os.path.join(windows10_sdk_dir, 'Include') - windows10sdk_kit_version_name = [x for x in os.listdir(windows10sdk_kits_include_dir) if os.path.isdir(os.path.join(windows10sdk_kits_include_dir, x))][0] # e.g. "10.0.10150.0" or "10.0.10240.0" + windows10sdk_kit_version_name = [ + x + for x in os.listdir(windows10sdk_kits_include_dir) + if os.path.isdir(os.path.join(windows10sdk_kits_include_dir, x)) + ][0] # e.g. "10.0.10150.0" or "10.0.10240.0" def append_item(key, item): if key not in env or len(env[key].strip()) == 0: diff --git a/test/common.py b/test/common.py index 7f5844adac982..f444adc1d3d41 100644 --- a/test/common.py +++ b/test/common.py @@ -191,6 +191,7 @@ def no_wasm64(note=''): def decorated(f): return skip_if(f, 'is_wasm64', note) + return decorated @@ -206,7 +207,9 @@ def decorated(self, *args, **kwargs): if self.get_setting('INITIAL_MEMORY') == '2200mb': self.skipTest(note) f(self, *args, **kwargs) + return decorated + return decorator @@ -221,7 +224,9 @@ def decorated(self, *args, **kwargs): if self.is_4gb(): self.skipTest(note) f(self, *args, **kwargs) + return decorated + return decorator @@ -328,6 +333,7 @@ def node_pthreads(f): def decorated(self, *args, **kwargs): self.setup_node_pthreads() f(self, *args, **kwargs) + return decorated @@ -366,6 +372,7 @@ def decorated(f): def modified(self, *args, **kwargs): with env_modify(updates): return f(self, *args, **kwargs) + return modified return decorated @@ -384,8 +391,7 @@ def metafunc(self, wasmfs, *args, **kwargs): self.emcc_args += ['-DMEMFS'] f(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'wasmfs': (True,)}) + parameterize(metafunc, {'': (False,), 'wasmfs': (True,)}) return metafunc @@ -464,14 +470,12 @@ def metafunc(self, rawfs, *args, **kwargs): self.emcc_args += ['-DMEMFS'] func(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'rawfs': (True,)}) + parameterize(metafunc, {'': (False,), 'rawfs': (True,)}) return metafunc # Decorator version of env_modify def also_with_env_modify(name_updates_mapping): - def decorated(f): @wraps(f) def metafunc(self, updates, *args, **kwargs): @@ -506,8 +510,7 @@ def metafunc(self, with_minimal_runtime, *args, **kwargs): self.set_setting('MINIMAL_RUNTIME', 1) f(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'minimal_runtime': (True,)}) + parameterize(metafunc, {'': (False,), 'minimal_runtime': (True,)}) return metafunc @@ -530,8 +533,7 @@ def metafunc(self, with_bigint, *args, **kwargs): else: f(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'bigint': (True,)}) + parameterize(metafunc, {'': (False,), 'bigint': (True,)}) return metafunc @@ -549,8 +551,7 @@ def metafunc(self, with_wasm64, *args, **kwargs): else: f(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'wasm64': (True,)}) + parameterize(metafunc, {'': (False,), 'wasm64': (True,)}) return metafunc @@ -569,8 +570,7 @@ def metafunc(self, with_wasm2js, *args, **kwargs): else: f(self, *args, **kwargs) - parameterize(metafunc, {'': (False,), - 'wasm2js': (True,)}) + parameterize(metafunc, {'': (False,), 'wasm2js': (True,)}) return metafunc @@ -583,11 +583,13 @@ def can_do_standalone(self, impure=False): # This is way to detect the core_2gb test mode in test_core.py if self.get_setting('INITIAL_MEMORY') == '2200mb': return False - return self.is_wasm() and \ - self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 and \ - not self.get_setting('MINIMAL_RUNTIME') and \ - not self.get_setting('SAFE_HEAP') and \ - not any(a.startswith('-fsanitize=') for a in self.emcc_args) + return ( + self.is_wasm() + and self.get_setting('STACK_OVERFLOW_CHECK', 0) < 2 + and not self.get_setting('MINIMAL_RUNTIME') + and not self.get_setting('SAFE_HEAP') + and not any(a.startswith('-fsanitize=') for a in self.emcc_args) + ) # Impure means a test that cannot run in a wasm VM yet, as it is not 100% @@ -618,8 +620,7 @@ def metafunc(self, standalone): self.node_args += shared.node_bigint_flags(nodejs) func(self) - parameterize(metafunc, {'': (False,), - 'standalone': (True,)}) + parameterize(metafunc, {'': (False,), 'standalone': (True,)}) return metafunc return decorated @@ -662,9 +663,7 @@ def metafunc(self, mode, *args, **kwargs): self.set_setting('DEFAULT_TO_CXX') f(self, *args, **kwargs) - parameterize(metafunc, {'emscripten': ('emscripten',), - 'wasm': ('wasm',), - 'wasm_exnref': ('wasm_exnref',)}) + parameterize(metafunc, {'emscripten': ('emscripten',), 'wasm': ('wasm',), 'wasm_exnref': ('wasm_exnref',)}) return metafunc @@ -692,9 +691,7 @@ def metafunc(self, mode, *args, **kwargs): self.set_setting('SUPPORT_LONGJMP', 'emscripten') f(self, *args, **kwargs) - parameterize(metafunc, {'emscripten': ('emscripten',), - 'wasm': ('wasm',), - 'wasm_exnref': ('wasm_exnref',)}) + parameterize(metafunc, {'emscripten': ('emscripten',), 'wasm': ('wasm',), 'wasm_exnref': ('wasm_exnref',)}) return metafunc @@ -716,11 +713,11 @@ def limit_size(string): if len(line) > max_line: lines[i] = line[:max_line] + '[..]' if len(lines) > maxlines: - lines = lines[0:maxlines // 2] + ['[..]'] + lines[-maxlines // 2:] + lines = lines[0 : maxlines // 2] + ['[..]'] + lines[-maxlines // 2 :] lines.append('(not all output shown. See `limit_size`)') string = '\n'.join(lines) + '\n' if len(string) > maxbytes: - string = string[0:maxbytes // 2] + '\n[..]\n' + string[-maxbytes // 2:] + string = string[0 : maxbytes // 2] + '\n[..]\n' + string[-maxbytes // 2 :] return string @@ -776,8 +773,7 @@ def force_delete_contents(dirname): def find_browser_test_file(filename): - """Looks for files in test/browser and then in test/ - """ + """Looks for files in test/browser and then in test/""" if not os.path.exists(filename): fullname = test_file('browser', filename) if not os.path.exists(fullname): @@ -797,7 +793,8 @@ def parameterize(func, parameters): if prev: # If we're parameterizing 2nd time, construct a cartesian product for various combinations. func._parameterize = { - '_'.join(filter(None, [k1, k2])): v2 + v1 for (k1, v1), (k2, v2) in itertools.product(prev.items(), parameters.items()) + '_'.join(filter(None, [k1, k2])): v2 + v1 + for (k1, v1), (k2, v2) in itertools.product(prev.items(), parameters.items()) } else: func._parameterize = parameters @@ -823,9 +820,11 @@ def test_something_subtest1(self): def test_something_subtest2(self): # runs test_something(4, 5, 6) """ + def decorator(func): parameterize(func, parameters) return func + return decorator @@ -1120,7 +1119,7 @@ def set_temp_dir(self, temp_dir): @classmethod def setUpClass(cls): super().setUpClass() - print('(checking sanity from test runner)') # do this after we set env stuff + print('(checking sanity from test runner)') # do this after we set env stuff shared.check_sanity(force=True) def setUp(self): @@ -1159,7 +1158,7 @@ def setUp(self): emcc_min_node_version = ( int(emcc_min_node_version_str[0:2]), int(emcc_min_node_version_str[2:4]), - int(emcc_min_node_version_str[4:6]) + int(emcc_min_node_version_str[4:6]), ) if node_version < emcc_min_node_version: self.emcc_args += building.get_emcc_node_flags(node_version) @@ -1225,16 +1224,19 @@ def tearDown(self): # They may not be due to us, but e.g. the browser when running browser # tests. Until we figure out a proper solution, ignore some temp file # names that we see on our CI infrastructure. - ignorable_file_prefixes = [ - '/tmp/tmpaddon', - '/tmp/circleci-no-output-timeout', - '/tmp/wasmer' - ] + ignorable_file_prefixes = ['/tmp/tmpaddon', '/tmp/circleci-no-output-timeout', '/tmp/wasmer'] left_over_files = set(temp_files_after_run) - set(self.temp_files_before_run) - left_over_files = [f for f in left_over_files if not any([f.startswith(prefix) for prefix in ignorable_file_prefixes])] + left_over_files = [ + f for f in left_over_files if not any([f.startswith(prefix) for prefix in ignorable_file_prefixes]) + ] if len(left_over_files): - print('ERROR: After running test, there are ' + str(len(left_over_files)) + ' new temporary files/directories left behind:', file=sys.stderr) + print( + 'ERROR: After running test, there are ' + + str(len(left_over_files)) + + ' new temporary files/directories left behind:', + file=sys.stderr, + ) for f in left_over_files: print('leaked file: ' + f, file=sys.stderr) self.fail('Test leaked ' + str(len(left_over_files)) + ' temporary files!') @@ -1295,7 +1297,17 @@ def add_on_exit(self, code): # libraries, for example def get_emcc_args(self, main_file=False, compile_only=False, asm_only=False): def is_ldflag(f): - return any(f.startswith(s) for s in ['-sEXPORT_ES6', '-sPROXY_TO_PTHREAD', '-sENVIRONMENT=', '--pre-js=', '--post-js=', '-sPTHREAD_POOL_SIZE=']) + return any( + f.startswith(s) + for s in [ + '-sEXPORT_ES6', + '-sPROXY_TO_PTHREAD', + '-sENVIRONMENT=', + '--pre-js=', + '--post-js=', + '-sPTHREAD_POOL_SIZE=', + ] + ) args = self.serialize_settings(compile_only or asm_only) + self.emcc_args if asm_only: @@ -1334,7 +1346,9 @@ def verify_es5(self, filename): self.fail('es-check failed to verify ES5 output compliance') # Build JavaScript code from source code - def build(self, filename, libraries=None, includes=None, force_c=False, js_outfile=True, emcc_args=None, output_basename=None): + def build( + self, filename, libraries=None, includes=None, force_c=False, js_outfile=True, emcc_args=None, output_basename=None + ): if not os.path.exists(filename): filename = test_file(filename) suffix = '.js' if js_outfile else '.wasm' @@ -1372,7 +1386,7 @@ def get_func(self, src, name): elif src[t] == '}': n -= 1 if n == 0: - return src[start:t + 1] + return src[start : t + 1] t += 1 assert t < len(src) @@ -1393,7 +1407,9 @@ def count_funcs(self, javascript_file): return num_funcs def count_wasm_contents(self, wasm_binary, what): - out = self.run_process([os.path.join(building.get_binaryen_bin(), 'wasm-opt'), wasm_binary, '--metrics'], stdout=PIPE).stdout + out = self.run_process( + [os.path.join(building.get_binaryen_bin(), 'wasm-opt'), wasm_binary, '--metrics'], stdout=PIPE + ).stdout # output is something like # [?] : 125 for line in out.splitlines(): @@ -1442,9 +1458,7 @@ def cleanup(line): assert len(long_lines) == 1 return '\n'.join(lines) - def run_js(self, filename, engine=None, args=None, - assert_returncode=0, - interleaved_output=True): + def run_js(self, filename, engine=None, args=None, assert_returncode=0, interleaved_output=True): # use files, as PIPE can get too full and hang us stdout_file = self.in_dir('stdout') stderr_file = None @@ -1463,10 +1477,7 @@ def run_js(self, filename, engine=None, args=None, if engine == config.V8_ENGINE: engine = engine + self.v8_args try: - jsrun.run_js(filename, engine, args, - stdout=stdout, - stderr=stderr, - assert_returncode=assert_returncode) + jsrun.run_js(filename, engine, args, stdout=stdout, stderr=stderr, assert_returncode=assert_returncode) except subprocess.TimeoutExpired as e: timeout_error = e except subprocess.CalledProcessError as e: @@ -1500,7 +1511,10 @@ def run_js(self, filename, engine=None, args=None, if assert_returncode == NON_ZERO: self.fail('JS subprocess unexpectedly succeeded (%s): Output:\n%s' % (error.cmd, ret)) else: - self.fail('JS subprocess failed (%s): %s (expected=%s). Output:\n%s' % (error.cmd, error.returncode, assert_returncode, ret)) + self.fail( + 'JS subprocess failed (%s): %s (expected=%s). Output:\n%s' + % (error.cmd, error.returncode, assert_returncode, ret) + ) # We should pass all strict mode checks self.assertNotContained('strict warning:', ret) @@ -1524,21 +1538,18 @@ def assertPathsIdentical(self, path1, path2): # Tests that the given two multiline text content are identical, modulo line # ending differences (\r\n on Windows, \n on Unix). - def assertTextDataIdentical(self, text1, text2, msg=None, - fromfile='expected', tofile='actual'): + def assertTextDataIdentical(self, text1, text2, msg=None, fromfile='expected', tofile='actual'): text1 = text1.replace('\r\n', '\n') text2 = text2.replace('\r\n', '\n') return self.assertIdentical(text1, text2, msg, fromfile, tofile) - def assertIdentical(self, values, y, msg=None, - fromfile='expected', tofile='actual'): + def assertIdentical(self, values, y, msg=None, fromfile='expected', tofile='actual'): if type(values) not in (list, tuple): values = [values] for x in values: if x == y: - return # success - diff_lines = difflib.unified_diff(x.splitlines(), y.splitlines(), - fromfile=fromfile, tofile=tofile) + return # success + diff_lines = difflib.unified_diff(x.splitlines(), y.splitlines(), fromfile=fromfile, tofile=tofile) diff = ''.join([a.rstrip() + '\n' for a in diff_lines]) if EMTEST_VERBOSE: print("Expected to have '%s' == '%s'" % (limit_size(values[0]), limit_size(y))) @@ -1566,12 +1577,10 @@ def assertFileContents(self, filename, contents): return if not os.path.exists(filename): - self.fail('Test expectation file not found: ' + filename + '.\n' + - 'Run with --rebaseline to generate.') + self.fail('Test expectation file not found: ' + filename + '.\n' + 'Run with --rebaseline to generate.') expected_content = read_file(filename) message = "Run with --rebaseline to automatically update expectations" - self.assertTextDataIdentical(expected_content, contents, message, - filename, filename + '.new') + self.assertTextDataIdentical(expected_content, contents, message, filename, filename + '.new') def assertContained(self, values, string, additional_info='', regex=False): if callable(string): @@ -1591,14 +1600,14 @@ def assertContained(self, values, string, additional_info='', regex=False): if not any(v in string for v in values): diff = difflib.unified_diff(values[0].split('\n'), string.split('\n'), fromfile='expected', tofile='actual') diff = ''.join(a.rstrip() + '\n' for a in diff) - self.fail("Expected to find '%s' in '%s', diff:\n\n%s\n%s" % ( - limit_size(values[0]), limit_size(string), limit_size(diff), - additional_info - )) + self.fail( + "Expected to find '%s' in '%s', diff:\n\n%s\n%s" + % (limit_size(values[0]), limit_size(string), limit_size(diff), additional_info) + ) def assertNotContained(self, value, string): if callable(value): - value = value() # lazy loading + value = value() # lazy loading if callable(string): string = string() if value in string: @@ -1611,10 +1620,8 @@ def assertContainedIf(self, value, string, condition): self.assertNotContained(value, string) def assertBinaryEqual(self, file1, file2): - self.assertEqual(os.path.getsize(file1), - os.path.getsize(file2)) - self.assertEqual(read_binary(file1), - read_binary(file2)) + self.assertEqual(os.path.getsize(file1), os.path.getsize(file2)) + self.assertEqual(read_binary(file1), read_binary(file2)) def check_expected_size_in_file(self, desc, filename, size): if EMTEST_REBASELINE: @@ -1623,7 +1630,9 @@ def check_expected_size_in_file(self, desc, filename, size): expected_size = int(read_file(filename).strip()) delta = size - expected_size ratio = abs(delta) / float(expected_size) - print(' seen %s size: %d (expected: %d) (delta: %d), ratio to expected: %f' % (desc, size, expected_size, delta, ratio)) + print( + ' seen %s size: %d (expected: %d) (delta: %d), ratio to expected: %f' % (desc, size, expected_size, delta, ratio) + ) self.assertLess(ratio, size_slack) library_cache: Dict[str, Tuple[str, object]] = {} @@ -1633,10 +1642,19 @@ def get_build_dir(self): ensure_dir(ret) return ret - def get_library(self, name, generated_libs, configure=['sh', './configure'], # noqa - configure_args=None, make=None, make_args=None, - env_init=None, cache_name_extra='', native=False, - force_rebuild=False): + def get_library( + self, + name, + generated_libs, + configure=['sh', './configure'], # noqa + configure_args=None, + make=None, + make_args=None, + env_init=None, + cache_name_extra='', + native=False, + force_rebuild=False, + ): if make is None: make = ['make'] if env_init is None: @@ -1655,7 +1673,13 @@ def get_library(self, name, generated_libs, configure=['sh', './configure'], # emcc_args = self.get_emcc_args(compile_only=True) hash_input = (str(emcc_args) + ' $ ' + str(env_init)).encode('utf-8') - cache_name = name + ','.join([opt for opt in emcc_args if len(opt) < 7]) + '_' + hashlib.md5(hash_input).hexdigest() + cache_name_extra + cache_name = ( + name + + ','.join([opt for opt in emcc_args if len(opt) < 7]) + + '_' + + hashlib.md5(hash_input).hexdigest() + + cache_name_extra + ) valid_chars = "_%s%s" % (string.ascii_letters, string.digits) cache_name = ''.join([(c if c in valid_chars else '_') for c in cache_name]) @@ -1678,9 +1702,19 @@ def get_library(self, name, generated_libs, configure=['sh', './configure'], # cflags = ' '.join(emcc_args) env_init.setdefault('CFLAGS', cflags) env_init.setdefault('CXXFLAGS', cflags) - return build_library(name, build_dir, output_dir, generated_libs, configure, - make, make_args, self.library_cache, - cache_name, env_init=env_init, native=native) + return build_library( + name, + build_dir, + output_dir, + generated_libs, + configure, + make, + make_args, + self.library_cache, + cache_name, + env_init=env_init, + native=native, + ) def clear(self): force_delete_contents(self.get_dir()) @@ -1739,7 +1773,9 @@ def expect_fail(self, cmd, expect_traceback=False, **args): # when run under browser it excercises how dynamic linker handles concurrency # - because B and C are loaded in parallel. def _test_dylink_dso_needed(self, do_run): - create_file('liba.cpp', r''' + create_file( + 'liba.cpp', + r''' #include #include @@ -1761,9 +1797,12 @@ def _test_dylink_dso_needed(self, do_run): }; static ainit _; - ''') + ''', + ) - create_file('libb.c', r''' + create_file( + 'libb.c', + r''' #include void afunc(const char *s); @@ -1771,9 +1810,12 @@ def _test_dylink_dso_needed(self, do_run): EMSCRIPTEN_KEEPALIVE void bfunc() { afunc("b"); } - ''') + ''', + ) - create_file('libc.c', r''' + create_file( + 'libc.c', + r''' #include void afunc(const char *s); @@ -1781,7 +1823,8 @@ def _test_dylink_dso_needed(self, do_run): EMSCRIPTEN_KEEPALIVE void cfunc() { afunc("c"); } - ''') + ''', + ) # _test_dylink_dso_needed can be potentially called several times by a test. # reset dylink-related options first. @@ -1806,7 +1849,8 @@ def ccshared(src, linkto=None): self.set_setting('MAIN_MODULE') extra_args = ['-L.', 'libb' + so, 'libc' + so] - do_run(r''' + do_run( + r''' #ifdef __cplusplus extern "C" { #endif @@ -1822,12 +1866,15 @@ def ccshared(src, linkto=None): return 0; } ''', - 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', emcc_args=extra_args) + 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', + emcc_args=extra_args, + ) extra_args = [] for libname in ('liba', 'libb', 'libc'): extra_args += ['--embed-file', libname + so] - do_run(r''' + do_run( + r''' #include #include #include @@ -1851,8 +1898,11 @@ def ccshared(src, linkto=None): cfunc_ptr(); return 0; } - ''' % locals(), - 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', emcc_args=extra_args) + ''' + % locals(), + 'a: loaded\na: b (prev: (null))\na: c (prev: b)\n', + emcc_args=extra_args, + ) def do_run(self, src, expected_output=None, force_c=False, **kwargs): if 'no_build' in kwargs: @@ -1883,23 +1933,37 @@ def do_run_in_out_file_test(self, srcfile, **kwargs): return output ## Does a complete test - builds, runs, checks output, etc. - def _build_and_run(self, filename, expected_output, args=None, - no_build=False, - libraries=None, - includes=None, - assert_returncode=0, assert_identical=False, assert_all=False, - check_for_error=True, force_c=False, emcc_args=None, - interleaved_output=True, - regex=False, - output_basename=None): + def _build_and_run( + self, + filename, + expected_output, + args=None, + no_build=False, + libraries=None, + includes=None, + assert_returncode=0, + assert_identical=False, + assert_all=False, + check_for_error=True, + force_c=False, + emcc_args=None, + interleaved_output=True, + regex=False, + output_basename=None, + ): logger.debug(f'_build_and_run: {filename}') if no_build: js_file = filename else: - js_file = self.build(filename, libraries=libraries, includes=includes, - force_c=force_c, emcc_args=emcc_args, - output_basename=output_basename) + js_file = self.build( + filename, + libraries=libraries, + includes=includes, + force_c=force_c, + emcc_args=emcc_args, + output_basename=output_basename, + ) self.assertExists(js_file) engines = self.js_engines.copy() @@ -1915,9 +1979,9 @@ def _build_and_run(self, filename, expected_output, args=None, if len(engines) == 0: self.fail('No JS engine present to run this test with. Check %s and the paths therein.' % config.EM_CONFIG) for engine in engines: - js_output = self.run_js(js_file, engine, args, - assert_returncode=assert_returncode, - interleaved_output=interleaved_output) + js_output = self.run_js( + js_file, engine, args, assert_returncode=assert_returncode, interleaved_output=interleaved_output + ) js_output = js_output.replace('\r\n', '\n') if expected_output: if type(expected_output) not in [list, tuple]: @@ -1947,16 +2011,18 @@ def get_freetype_library(self): # And because gnu-offsetof-extensions is a new warning: '-Wno-unknown-warning-option', ] - return self.get_library(os.path.join('third_party', 'freetype'), - os.path.join('objs', '.libs', 'libfreetype.a'), - configure_args=['--disable-shared', '--without-zlib']) + return self.get_library( + os.path.join('third_party', 'freetype'), + os.path.join('objs', '.libs', 'libfreetype.a'), + configure_args=['--disable-shared', '--without-zlib'], + ) def get_poppler_library(self, env_init=None): freetype = self.get_freetype_library() self.emcc_args += [ '-I' + test_file('third_party/freetype/include'), - '-I' + test_file('third_party/poppler/include') + '-I' + test_file('third_party/poppler/include'), ] # Poppler has some pretty glaring warning. Suppress them to keep the @@ -1982,10 +2048,24 @@ def get_poppler_library(self, env_init=None): env_init['FONTCONFIG_LIBS'] = ' ' poppler = self.get_library( - os.path.join('third_party', 'poppler'), - [os.path.join('utils', 'pdftoppm.o'), os.path.join('utils', 'parseargs.o'), os.path.join('poppler', '.libs', 'libpoppler.a')], - env_init=env_init, - configure_args=['--disable-libjpeg', '--disable-libpng', '--disable-poppler-qt', '--disable-poppler-qt4', '--disable-cms', '--disable-cairo-output', '--disable-abiword-output', '--disable-shared']) + os.path.join('third_party', 'poppler'), + [ + os.path.join('utils', 'pdftoppm.o'), + os.path.join('utils', 'parseargs.o'), + os.path.join('poppler', '.libs', 'libpoppler.a'), + ], + env_init=env_init, + configure_args=[ + '--disable-libjpeg', + '--disable-libpng', + '--disable-poppler-qt', + '--disable-poppler-qt4', + '--disable-cms', + '--disable-cairo-output', + '--disable-abiword-output', + '--disable-shared', + ], + ) return poppler + freetype @@ -2001,10 +2081,13 @@ def get_zlib_library(self, cmake): # https://github.com/emscripten-core/emscripten/issues/16908 is fixed self.emcc_args.append('-Wno-pointer-sign') if cmake: - rtn = self.get_library(os.path.join('third_party', 'zlib'), os.path.join('libz.a'), - configure=['cmake', '.'], - make=['cmake', '--build', '.', '--'], - make_args=[]) + rtn = self.get_library( + os.path.join('third_party', 'zlib'), + os.path.join('libz.a'), + configure=['cmake', '.'], + make=['cmake', '--build', '.', '--'], + make_args=[], + ) else: rtn = self.get_library(os.path.join('third_party', 'zlib'), os.path.join('libz.a'), make_args=['libz.a']) self.emcc_args = old_args @@ -2140,7 +2223,7 @@ def log_request(code=0, size=0): SimpleHTTPRequestHandler.extensions_map['.wasm'] = 'application/wasm' httpd = HTTPServer(('localhost', port), TestServerHandler) - httpd.serve_forever() # test runner will kill us + httpd.serve_forever() # test runner will kill us class Reporting(Enum): @@ -2148,6 +2231,7 @@ class Reporting(Enum): code for reporting results back to the browser. This enum allows tests to decide what type of support code they need/want. """ + NONE = 0 # Include the JS helpers for reporting results JS_ONLY = 1 @@ -2202,7 +2286,9 @@ def setUpClass(cls): return cls.harness_in_queue = multiprocessing.Queue() cls.harness_out_queue = multiprocessing.Queue() - cls.harness_server = multiprocessing.Process(target=harness_server_func, args=(cls.harness_in_queue, cls.harness_out_queue, cls.PORT)) + cls.harness_server = multiprocessing.Process( + target=harness_server_func, args=(cls.harness_in_queue, cls.harness_out_queue, cls.PORT) + ) cls.harness_server.start() print('[Browser harness server on process %d]' % cls.harness_server.pid) cls.browser_open(cls.HARNESS_URL) @@ -2246,10 +2332,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr assert not (message and expected), 'run_browser expects `expected` or `message`, but not both' if expected is not None: try: - self.harness_in_queue.put(( - 'http://localhost:%s/%s' % (self.PORT, html_file), - self.get_dir() - )) + self.harness_in_queue.put(('http://localhost:%s/%s' % (self.PORT, html_file), self.get_dir())) if timeout is None: timeout = self.BROWSER_TIMEOUT try: @@ -2266,7 +2349,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr # us to also fail the test self.fail('browser harness error') if output.startswith('/report_result?skipped:'): - self.skipTest(unquote(output[len('/report_result?skipped:'):]).strip()) + self.skipTest(unquote(output[len('/report_result?skipped:') :]).strip()) else: # verify the result, and try again if we should do so output = unquote(output) @@ -2280,7 +2363,7 @@ def run_browser(self, html_file, expected=None, message=None, timeout=None, extr else: raise e finally: - time.sleep(0.1) # see comment about Windows above + time.sleep(0.1) # see comment about Windows above self.assert_out_queue_empty('this test') else: webbrowser.open_new(os.path.abspath(html_file)) @@ -2301,8 +2384,11 @@ def compile_btest(self, filename, args, reporting=Reporting.FULL): if reporting == Reporting.FULL: # If C reporting (i.e. the REPORT_RESULT macro) is required we # also include report_result.c and force-include report_result.h - self.run_process([EMCC, '-c', '-I' + TEST_ROOT, - test_file('report_result.c')] + self.get_emcc_args(compile_only=True) + (['-fPIC'] if '-fPIC' in args else [])) + self.run_process( + [EMCC, '-c', '-I' + TEST_ROOT, test_file('report_result.c')] + + self.get_emcc_args(compile_only=True) + + (['-fPIC'] if '-fPIC' in args else []) + ) args += ['report_result.o', '-include', test_file('report_result.h')] if EMTEST_BROWSER == 'node': args.append('-DEMTEST_NODE') @@ -2324,12 +2410,18 @@ def btest_exit(self, filename, assert_returncode=0, *args, **kwargs): kwargs['expected'] = 'exit:%d' % assert_returncode return self.btest(filename, *args, **kwargs) - def btest(self, filename, expected=None, - post_build=None, - args=None, url_suffix='', timeout=None, - extra_tries=1, - reporting=Reporting.FULL, - output_basename='test'): + def btest( + self, + filename, + expected=None, + post_build=None, + args=None, + url_suffix='', + timeout=None, + extra_tries=1, + reporting=Reporting.FULL, + output_basename='test', + ): assert expected, 'a btest must have an expected output' if args is None: args = [] @@ -2356,23 +2448,20 @@ def btest(self, filename, expected=None, output = self.run_js('test.js') self.assertContained('RESULT: ' + expected[0], output) else: - self.run_browser(outfile + url_suffix, expected=['/report_result?' + e for e in expected], timeout=timeout, extra_tries=extra_tries) + self.run_browser( + outfile + url_suffix, + expected=['/report_result?' + e for e in expected], + timeout=timeout, + extra_tries=extra_tries, + ) ################################################################################################### -def build_library(name, - build_dir, - output_dir, - generated_libs, - configure, - make, - make_args, - cache, - cache_name, - env_init, - native): +def build_library( + name, build_dir, output_dir, generated_libs, configure, make, make_args, cache, cache_name, env_init, native +): """Build a library and cache the result. We build the library file once and cache it for all our tests. (We cache in memory since the test directory is destroyed and recreated for each test. Note that we cache @@ -2415,8 +2504,7 @@ def build_library(name, with open(os.path.join(project_dir, 'configure_err'), 'w') as err: stdout = out if EMTEST_BUILD_VERBOSE < 2 else None stderr = err if EMTEST_BUILD_VERBOSE < 1 else None - shared.run_process(configure, env=env, stdout=stdout, stderr=stderr, - cwd=project_dir) + shared.run_process(configure, env=env, stdout=stdout, stderr=stderr, cwd=project_dir) except subprocess.CalledProcessError: print('-- configure stdout --') print(read_file(Path(project_dir, 'configure_out'))) @@ -2444,8 +2532,7 @@ def open_make_err(mode='r'): with open_make_err('w') as make_err: stdout = make_out if EMTEST_BUILD_VERBOSE < 2 else None stderr = make_err if EMTEST_BUILD_VERBOSE < 1 else None - shared.run_process(make + make_args, stdout=stdout, stderr=stderr, env=env, - cwd=project_dir) + shared.run_process(make + make_args, stdout=stdout, stderr=stderr, env=env, cwd=project_dir) except subprocess.CalledProcessError: with open_make_out() as f: print('-- make stdout --') diff --git a/test/gen_large_switchcase.py b/test/gen_large_switchcase.py index e9f5cef0825f2..e8883a903108f 100755 --- a/test/gen_large_switchcase.py +++ b/test/gen_large_switchcase.py @@ -15,7 +15,8 @@ i += incr incr = (incr % 5) + 1 -print('''#include +print( + '''#include #include #include @@ -23,14 +24,19 @@ { switch(x) { -''' + cases + ''' +''' + + cases + + ''' default: return "default"; } } int main() { - for(int i = 0; i < ''' + str((num_cases + 99) // 100) + '''; ++i) + for(int i = 0; i < ''' + + str((num_cases + 99) // 100) + + '''; ++i) printf("%d: %s\\n", i*301, foo(i*301)); printf("Success!\\n"); -}''') +}''' +) diff --git a/test/gen_many_js_functions.py b/test/gen_many_js_functions.py index dccc74aca2b29..80138cf448796 100644 --- a/test/gen_many_js_functions.py +++ b/test/gen_many_js_functions.py @@ -11,7 +11,10 @@ def func_name(i): - return 'thisIsAFunctionWithVeryLongFunctionNameThatWouldBeGreatToBeMinifiedWhenImportingToAsmJsOrWasmSideCodeToCallOtherwiseCodeSizesWillBeLargeAndNetworkTransfersBecomeVerySlowThatUsersWillGoAwayAndVisitSomeOtherSiteInsteadAndThenWebAssemblyDeveloperIsSadOrEvenWorseNobodyNoticesButInternetPipesWillGetMoreCongestedWhichContributesToGlobalWarmingAndThenEveryoneElseWillBeSadAsWellEspeciallyThePolarBearsAndPenguinsJustThinkAboutThePenguins' + str(i + 1) + return ( + 'thisIsAFunctionWithVeryLongFunctionNameThatWouldBeGreatToBeMinifiedWhenImportingToAsmJsOrWasmSideCodeToCallOtherwiseCodeSizesWillBeLargeAndNetworkTransfersBecomeVerySlowThatUsersWillGoAwayAndVisitSomeOtherSiteInsteadAndThenWebAssemblyDeveloperIsSadOrEvenWorseNobodyNoticesButInternetPipesWillGetMoreCongestedWhichContributesToGlobalWarmingAndThenEveryoneElseWillBeSadAsWellEspeciallyThePolarBearsAndPenguinsJustThinkAboutThePenguins' + + str(i + 1) + ) def generate_js_library_with_lots_of_functions(out_file): @@ -38,7 +41,13 @@ def generate_c_program_that_calls_js_library_with_lots_of_functions(out_file): for i in range(NUM_FUNCS_TO_GENERATE): f.write(' sum += ' + func_name(i) + '();\n') - f.write('\n printf("Sum of numbers from 1 to ' + str(NUM_FUNCS_TO_GENERATE) + ': %d (expected ' + str(int((NUM_FUNCS_TO_GENERATE * (NUM_FUNCS_TO_GENERATE + 1)) / 2)) + ')\\n", sum);\n') + f.write( + '\n printf("Sum of numbers from 1 to ' + + str(NUM_FUNCS_TO_GENERATE) + + ': %d (expected ' + + str(int((NUM_FUNCS_TO_GENERATE * (NUM_FUNCS_TO_GENERATE + 1)) / 2)) + + ')\\n", sum);\n' + ) f.write('}\n') diff --git a/test/jsrun.py b/test/jsrun.py index b918becaed8e0..8f8a58d8e783a 100644 --- a/test/jsrun.py +++ b/test/jsrun.py @@ -12,7 +12,7 @@ from tools import shared, utils -WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False +WORKING_ENGINES = {} # Holds all configured engines and whether they work: maps path -> True/False DEFAULT_TIMEOUT = 5 * 60 @@ -83,10 +83,19 @@ def require_engine(engine): sys.exit(1) -def run_js(filename, engine, args=None, - stdin=None, stdout=PIPE, stderr=None, cwd=None, - full_output=False, assert_returncode=0, skip_check=False, - timeout=DEFAULT_TIMEOUT): +def run_js( + filename, + engine, + args=None, + stdin=None, + stdout=PIPE, + stderr=None, + cwd=None, + full_output=False, + assert_returncode=0, + skip_check=False, + timeout=DEFAULT_TIMEOUT, +): """Execute javascript code generated by tests, with possible timeout.""" # We used to support True here but we no longer do. Assert here just in case. @@ -100,13 +109,8 @@ def run_js(filename, engine, args=None, print(f"Running: '{shared.shlex_join(command)}'") try: proc = subprocess.run( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - timeout=timeout, - universal_newlines=True) + command, stdin=stdin, stdout=stdout, stderr=stderr, cwd=cwd, timeout=timeout, universal_newlines=True + ) except Exception: # the failure may be because the engine is not present. show the proper # error in that case diff --git a/test/other/ports/external.py b/test/other/ports/external.py index 5fb8c8d764f4f..2c5f57915bbad 100644 --- a/test/other/ports/external.py +++ b/test/other/ports/external.py @@ -18,12 +18,7 @@ } # user options (from --use-port) -opts: Dict[str, Optional[str]] = { - 'value1': None, - 'value2': None, - 'value3': "v3", - 'dependency': None -} +opts: Dict[str, Optional[str]] = {'value1': None, 'value2': None, 'value3': "v3", 'dependency': None} deps = ['sdl2_image:formats=jpg'] diff --git a/test/parallel_testsuite.py b/test/parallel_testsuite.py index 05f3717f47f83..013d0490835cc 100644 --- a/test/parallel_testsuite.py +++ b/test/parallel_testsuite.py @@ -91,11 +91,12 @@ def combine_results(self, result, buffered_results): return result -class BufferedParallelTestResult(): +class BufferedParallelTestResult: """A picklable struct used to communicate test results across processes Fulfills the interface for unittest.TestResult """ + def __init__(self): self.buffered_result = None @@ -145,8 +146,9 @@ def addError(self, test, err): self.buffered_result = BufferedTestError(test, err) -class BufferedTestBase(): +class BufferedTestBase: """Abstract class that holds test result data, split by type of result.""" + def __init__(self, test, err=None): self.test = test if err: @@ -181,6 +183,7 @@ def fixup_fake_exception(fake_exc): # the data def make_wrapper(rtn): return lambda: rtn + ex.tb_frame.f_code.co_positions = make_wrapper(ex.tb_frame.f_code.positions) ex = ex.tb_next @@ -209,7 +212,7 @@ def updateResult(self, result): result.addUnexpectedSuccess(self.test) -class FakeTraceback(): +class FakeTraceback: """A fake version of a traceback object that is picklable across processes. Python's traceback objects contain hidden stack information that isn't able @@ -228,14 +231,14 @@ def __init__(self, tb): self.tb_lasti = tb.tb_lasti -class FakeFrame(): +class FakeFrame: def __init__(self, f): self.f_code = FakeCode(f.f_code) # f.f_globals is not picklable, not used in stack traces, and needs to be iterable self.f_globals = [] -class FakeCode(): +class FakeCode: def __init__(self, co): self.co_filename = co.co_filename self.co_name = co.co_name diff --git a/test/runner.py b/test/runner.py index 5896fe4f15b9e..5ae1bacbafca6 100755 --- a/test/runner.py +++ b/test/runner.py @@ -135,7 +135,7 @@ def get_all_tests(modules): def get_crossplatform_tests(modules): - suites = ['core0', 'other', 'sanity'] # We don't need all versions of every test + suites = ['core0', 'other', 'sanity'] # We don't need all versions of every test crossplatform_tests = [] # Walk over the test suites and find the test functions with the # is_crossplatform_test attribute applied by @crossplatform decorator @@ -248,15 +248,18 @@ def print_random_test_statistics(num_tests): std = 0.5 / math.sqrt(num_tests) expected = 100.0 * (1.0 - std) print() - print('running those %d randomly-selected tests. if they all pass, then there is a ' - 'greater than 95%% chance that at least %.2f%% of the test suite will pass' - % (num_tests, expected)) + print( + 'running those %d randomly-selected tests. if they all pass, then there is a ' + 'greater than 95%% chance that at least %.2f%% of the test suite will pass' % (num_tests, expected) + ) print() def show(): - print('if all tests passed then there is a greater than 95%% chance that at least ' - '%.2f%% of the test suite will pass' - % (expected)) + print( + 'if all tests passed then there is a greater than 95%% chance that at least ' + '%.2f%% of the test suite will pass' % (expected) + ) + atexit.register(show) @@ -314,6 +317,7 @@ def flattened_tests(loaded_tests): tests.extend(subsuite) return tests + def suite_for_module(module, tests): suite_supported = module.__name__ in ('test_core', 'test_other', 'test_posixtest') if not common.EMTEST_SAVE_DIR and not shared.DEBUG: @@ -338,9 +342,9 @@ def run_tests(options, suites): os.makedirs('out', exist_ok=True) # output fd must remain open until after testRunner.run() below output = open('out/test-results.xml', 'wb') - import xmlrunner # type: ignore - testRunner = xmlrunner.XMLTestRunner(output=output, verbosity=2, - failfast=options.failfast) + import xmlrunner # type: ignore + + testRunner = xmlrunner.XMLTestRunner(output=output, verbosity=2, failfast=options.failfast) print('Writing XML test output to ' + os.path.abspath(output.name)) else: testRunner = unittest.TextTestRunner(verbosity=2, failfast=options.failfast) @@ -348,8 +352,13 @@ def run_tests(options, suites): for mod_name, suite in suites: print('Running %s: (%s tests)' % (mod_name, suite.countTestCases())) res = testRunner.run(suite) - msg = ('%s: %s run, %s errors, %s failures, %s skipped' % - (mod_name, res.testsRun, len(res.errors), len(res.failures), len(res.skipped))) + msg = '%s: %s run, %s errors, %s failures, %s skipped' % ( + mod_name, + res.testsRun, + len(res.errors), + len(res.failures), + len(res.skipped), + ) num_failures += len(res.errors) + len(res.failures) + len(res.unexpectedSuccesses) resultMessages.append(msg) @@ -365,32 +374,41 @@ def run_tests(options, suites): def parse_args(args): parser = argparse.ArgumentParser(prog='runner.py', description=__doc__) - parser.add_argument('--save-dir', action='store_true', - help='Save the temporary directory used during for each ' - 'test. Implies --cores=1. Defaults to true when running a single test') - parser.add_argument('--no-clean', action='store_true', - help='Do not clean the temporary directory before each test run') + parser.add_argument( + '--save-dir', + action='store_true', + help='Save the temporary directory used during for each ' + 'test. Implies --cores=1. Defaults to true when running a single test', + ) + parser.add_argument( + '--no-clean', action='store_true', help='Do not clean the temporary directory before each test run' + ) parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--all-engines', action='store_true') parser.add_argument('--detect-leaks', action='store_true') parser.add_argument('--skip-slow', action='store_true', help='Skip tests marked as slow') - parser.add_argument('--cores', '-j', - help='Set the number tests to run in parallel. Defaults ' - 'to the number of CPU cores.', default=None) - parser.add_argument('--rebaseline', action='store_true', - help='Automatically update test expectations for tests that support it.') - parser.add_argument('--browser', - help='Command to launch web browser in which to run browser tests.') + parser.add_argument( + '--cores', + '-j', + help='Set the number tests to run in parallel. Defaults ' 'to the number of CPU cores.', + default=None, + ) + parser.add_argument( + '--rebaseline', action='store_true', help='Automatically update test expectations for tests that support it.' + ) + parser.add_argument('--browser', help='Command to launch web browser in which to run browser tests.') parser.add_argument('tests', nargs='*') parser.add_argument('--failfast', action='store_true') parser.add_argument('--start-at', metavar='NAME', help='Skip all tests up until ') - parser.add_argument('--continue', dest='_continue', action='store_true', - help='Resume from the last run test.' - 'Useful when combined with --failfast') + parser.add_argument( + '--continue', + dest='_continue', + action='store_true', + help='Resume from the last run test.' 'Useful when combined with --failfast', + ) parser.add_argument('--force64', action='store_true') parser.add_argument('--crossplatform-only', action='store_true') - parser.add_argument('--repeat', type=int, default=1, - help='Repeat each test N times (default: 1).') + parser.add_argument('--repeat', type=int, default=1, help='Repeat each test N times (default: 1).') return parser.parse_args() diff --git a/test/test_benchmark.py b/test/test_benchmark.py index c0c615fc138e5..bdca3c8138a20 100644 --- a/test/test_benchmark.py +++ b/test/test_benchmark.py @@ -56,7 +56,7 @@ EMTEST_BENCHMARKERS = os.getenv('EMTEST_BENCHMARKERS', 'clang,v8,v8-lto,v8-ctors') -class Benchmarker(): +class Benchmarker: # Whether to record statistics. Set by SizeBenchmarker. record_stats = False @@ -78,7 +78,7 @@ def bench(self, args, output_parser=None, reps=TEST_REPS, expected_output=None): if expected_output is not None and expected_output not in output: raise ValueError('Incorrect benchmark output:\n' + output) - if not output_parser or args == ['0']: # if arg is 0, we are not running code, and have no output to parse + if not output_parser or args == ['0']: # if arg is 0, we are not running code, and have no output to parse curr = time.time() - start else: try: @@ -101,11 +101,15 @@ def display(self, baseline=None): sorted_times = sorted(self.times) count = len(sorted_times) if count % 2 == 0: - median = sum(sorted_times[count // 2 - 1:count // 2 + 1]) / 2 + median = sum(sorted_times[count // 2 - 1 : count // 2 + 1]) / 2 else: median = sorted_times[count // 2] - print(' %10s: mean: %4.3f (+-%4.3f) secs median: %4.3f range: %4.3f-%4.3f (noise: %4.3f%%) (%d runs)' % (self.name, mean, std, median, min(self.times), max(self.times), 100 * std / mean, self.reps), end=' ') + print( + ' %10s: mean: %4.3f (+-%4.3f) secs median: %4.3f range: %4.3f-%4.3f (noise: %4.3f%%) (%d runs)' + % (self.name, mean, std, median, min(self.times), max(self.times), 100 * std / mean, self.reps), + end=' ', + ) if baseline: mean_baseline = sum(baseline.times) / len(baseline.times) @@ -119,14 +123,18 @@ def display(self, baseline=None): recorded_stats = [] def add_stat(name, size, gzip_size): - recorded_stats.append({ - 'value': name, - 'measurement': size, - }) - recorded_stats.append({ - 'value': name + ' (gzipped)', - 'measurement': gzip_size, - }) + recorded_stats.append( + { + 'value': name, + 'measurement': size, + } + ) + recorded_stats.append( + { + 'value': name + ' (gzipped)', + 'measurement': gzip_size, + } + ) total_size = 0 total_gzip_size = 0 @@ -160,7 +168,9 @@ def __init__(self, name, cc, cxx, args=None): self.cxx = cxx self.args = args or [OPTIMIZATIONS] - def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): + def build( + self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser + ): native_args = native_args or [] shared_args = shared_args or [] self.parent = parent @@ -170,11 +180,14 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat native_args = native_args + lib_builder(self.name, native=True, env_init=env) if not native_exec: compiler = self.cxx if filename.endswith('cpp') else self.cc - cmd = compiler + [ - '-fno-math-errno', - filename, - '-o', filename + '.native' - ] + self.args + shared_args + native_args + clang_native.get_clang_native_args() + cmd = ( + compiler + + ['-fno-math-errno', filename, '-o', filename + '.native'] + + self.args + + shared_args + + native_args + + clang_native.get_clang_native_args() + ) # print(cmd) run_process(cmd, env=clang_native.get_clang_native_env()) else: @@ -196,11 +209,9 @@ def get_size_text(self): def run_binaryen_opts(filename, opts): - run_process([ - os.path.join(building.get_binaryen_bin(), 'wasm-opt', '--all-features'), - filename, - '-o', filename - ] + opts) + run_process( + [os.path.join(building.get_binaryen_bin(), 'wasm-opt', '--all-features'), filename, '-o', filename] + opts + ) class EmscriptenBenchmarker(Benchmarker): @@ -213,7 +224,9 @@ def __init__(self, name, engine, extra_args=None, env=None, binaryen_opts=None): self.env.update(env) self.binaryen_opts = binaryen_opts or [] - def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): + def build( + self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser + ): emcc_args = emcc_args or [] self.filename = filename llvm_root = self.env.get('LLVM') or config.LLVM_ROOT @@ -226,15 +239,24 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat # This shouldn't be 'emcc_args += ...', because emcc_args is passed in as # a parameter and changes will be visible to the caller. emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=env_init) - final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js' + final = ( + os.path.dirname(filename) + + os.path.sep + + self.name + + ('_' if self.name else '') + + os.path.basename(filename) + + '.js' + ) final = final.replace('.cpp', '') utils.delete_file(final) cmd = [ - EMCC, filename, + EMCC, + filename, OPTIMIZATIONS, '-sINITIAL_MEMORY=256MB', '-sENVIRONMENT=node,shell', - '-o', final + '-o', + final, ] + LLVM_FEATURE_FLAGS if shared_args: cmd += shared_args @@ -294,7 +316,9 @@ def __init__(self, name, engine, args=None, binaryen_opts=None): self.args = args or [OPTIMIZATIONS] self.binaryen_opts = binaryen_opts or [] - def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser): + def build( + self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser + ): cheerp_args = [ '-fno-math-errno', ] @@ -303,20 +327,31 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat if lib_builder: # build as "native" (so no emcc env stuff), but with all the cheerp stuff # set in the env - cheerp_args = cheerp_args + lib_builder(self.name, native=True, env_init={ - 'CC': CHEERP_BIN + 'clang', - 'CXX': CHEERP_BIN + 'clang++', - 'AR': CHEERP_BIN + '../libexec/cheerp-unknown-none-ar', - 'LD': CHEERP_BIN + 'clang', - 'NM': CHEERP_BIN + 'llvm-nm', - 'LDSHARED': CHEERP_BIN + 'clang', - 'RANLIB': CHEERP_BIN + '../libexec/cheerp-unknown-none-ranlib', - 'CXXFLAGS': "-Wno-c++11-narrowing", - 'CHEERP_PREFIX': CHEERP_BIN + '../', - }) + cheerp_args = cheerp_args + lib_builder( + self.name, + native=True, + env_init={ + 'CC': CHEERP_BIN + 'clang', + 'CXX': CHEERP_BIN + 'clang++', + 'AR': CHEERP_BIN + '../libexec/cheerp-unknown-none-ar', + 'LD': CHEERP_BIN + 'clang', + 'NM': CHEERP_BIN + 'llvm-nm', + 'LDSHARED': CHEERP_BIN + 'clang', + 'RANLIB': CHEERP_BIN + '../libexec/cheerp-unknown-none-ranlib', + 'CXXFLAGS': "-Wno-c++11-narrowing", + 'CHEERP_PREFIX': CHEERP_BIN + '../', + }, + ) if PROFILING: - cheerp_args += ['-cheerp-pretty-code'] # get function names, like emcc --profiling - final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js' + cheerp_args += ['-cheerp-pretty-code'] # get function names, like emcc --profiling + final = ( + os.path.dirname(filename) + + os.path.sep + + self.name + + ('_' if self.name else '') + + os.path.basename(filename) + + '.js' + ) final = final.replace('.cpp', '') utils.delete_file(final) dirs_to_delete = [] @@ -327,12 +362,18 @@ def build(self, parent, filename, args, shared_args, emcc_args, native_args, nat compiler = CHEERP_BIN + '/clang' else: compiler = CHEERP_BIN + '/clang++' - cmd = [compiler] + cheerp_args + [ - '-cheerp-linear-heap-size=256', - '-cheerp-secondary-output-file=' + final.replace('.js', '.wasm'), - filename, - '-o', final - ] + shared_args + cmd = ( + [compiler] + + cheerp_args + + [ + '-cheerp-linear-heap-size=256', + '-cheerp-secondary-output-file=' + final.replace('.js', '.wasm'), + filename, + '-o', + final, + ] + + shared_args + ) # print(' '.join(cmd)) run_process(cmd, stdout=PIPE, stderr=PIPE) self.filename = final @@ -360,7 +401,7 @@ def get_output_files(self): named_benchmarkers = { 'clang': NativeBenchmarker('clang', [CLANG_CC], [CLANG_CXX]), - 'gcc': NativeBenchmarker('gcc', ['gcc', '-no-pie'], ['g++', '-no-pie']), + 'gcc': NativeBenchmarker('gcc', ['gcc', '-no-pie'], ['g++', '-no-pie']), 'size': SizeBenchmarker('size'), 'v8': EmscriptenBenchmarker('v8', aot_v8), 'v8-lto': EmscriptenBenchmarker('v8-lto', aot_v8, ['-flto']), @@ -371,7 +412,7 @@ def get_output_files(self): 'cherp-v8': CheerpBenchmarker('cheerp-v8-wasm', aot_v8), # TODO: ensure no baseline compiler is used, see v8 'sm': EmscriptenBenchmarker('sm', config.SPIDERMONKEY_ENGINE), - 'cherp-sm': CheerpBenchmarker('cheerp-sm-wasm', config.SPIDERMONKEY_ENGINE) + 'cherp-sm': CheerpBenchmarker('cheerp-sm-wasm', config.SPIDERMONKEY_ENGINE), } for name in EMTEST_BENCHMARKERS.split(','): @@ -382,7 +423,7 @@ def get_output_files(self): class benchmark(common.RunnerCore): save_dir = True - stats = [] # type: ignore + stats = [] # type: ignore @classmethod def setUpClass(cls): @@ -398,7 +439,10 @@ def setUpClass(cls): pass try: with common.chdir(os.path.expanduser('~/Dev/mozilla-central')): - fingerprint.append('sm: ' + [line for line in run_process(['hg', 'tip'], stdout=PIPE).stdout.splitlines() if 'changeset' in line][0]) + fingerprint.append( + 'sm: ' + + [line for line in run_process(['hg', 'tip'], stdout=PIPE).stdout.splitlines() if 'changeset' in line][0] + ) except Exception: pass fingerprint.append('llvm: ' + config.LLVM_ROOT) @@ -408,11 +452,7 @@ def setUpClass(cls): def tearDownClass(cls): super().tearDownClass() if cls.stats: - output = { - 'version': 1, - 'git_hash': '', - 'results': cls.stats - } + output = {'version': 1, 'git_hash': '', 'results': cls.stats} utils.write_file('stats.json', json.dumps(output, indent=2) + '\n') # avoid depending on argument reception from the commandline @@ -422,21 +462,36 @@ def hardcode_arguments(self, code): main_pattern = 'int main(int argc, char **argv)' assert main_pattern in code code = code.replace(main_pattern, 'int benchmark_main(int argc, char **argv)') - code += ''' + code += ( + ''' int main() { int newArgc = 2; char* newArgv[] = { (char*)"./program.exe", (char*)"%s" }; int ret = benchmark_main(newArgc, newArgv); return ret; } - ''' % DEFAULT_ARG + ''' + % DEFAULT_ARG + ) return code - def do_benchmark(self, name, src, expected_output='FAIL', args=None, - emcc_args=None, native_args=None, shared_args=None, - force_c=False, reps=TEST_REPS, native_exec=None, - output_parser=None, args_processor=None, lib_builder=None, - skip_native=False): + def do_benchmark( + self, + name, + src, + expected_output='FAIL', + args=None, + emcc_args=None, + native_args=None, + shared_args=None, + force_c=False, + reps=TEST_REPS, + native_exec=None, + output_parser=None, + args_processor=None, + lib_builder=None, + skip_native=False, + ): if not benchmarkers: raise Exception('error, no benchmarkers') @@ -459,22 +514,24 @@ def do_benchmark(self, name, src, expected_output='FAIL', args=None, reps = 0 baseline = b print('Running benchmarker: %s: %s' % (b.__class__.__name__, b.name)) - b.build(self, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser=output_parser is not None) + b.build( + self, + filename, + args, + shared_args, + emcc_args, + native_args, + native_exec, + lib_builder, + has_output_parser=output_parser is not None, + ) b.bench(args, output_parser, reps, expected_output) recorded_stats = b.display(baseline) if recorded_stats: self.add_stats(name, recorded_stats) def add_stats(self, name, stats): - self.stats.append({ - 'key': { - 'test': name, - 'units': 'bytes' - }, - 'measurements': { - 'stats': stats - } - }) + self.stats.append({'key': {'test': name, 'units': 'bytes'}, 'measurements': {'stats': stats}}) def test_primes(self, check=True): src = r''' @@ -516,7 +573,12 @@ def test_primes(self, check=True): return 0; } ''' - self.do_benchmark('primes' if check else 'primes-nocheck', src, 'lastprime:' if check else '', shared_args=['-DCHECK'] if check else []) + self.do_benchmark( + 'primes' if check else 'primes-nocheck', + src, + 'lastprime:' if check else '', + shared_args=['-DCHECK'] if check else [], + ) # Also interesting to test it without the printfs which allow checking the output. Without # printf, code size is dominated by the runtime itself (the compiled code is just a few lines). @@ -750,7 +812,7 @@ def test_fannkuch(self): case 5: n = 12; break; default: printf("error: %d\\n", arg); return -1; } - ''' + ''', ) assert 'switch(arg)' in src self.do_benchmark('fannkuch', src, 'Pfannkuchen(') @@ -824,7 +886,9 @@ def test_corrections64(self): def fasta(self, name, double_rep): src = read_file(test_file('fasta.cpp')).replace('double', double_rep) - src = src.replace(' const size_t n = ( argc > 1 ) ? atoi( argv[1] ) : 512;', ''' + src = src.replace( + ' const size_t n = ( argc > 1 ) ? atoi( argv[1] ) : 512;', + ''' int n; int arg = argc > 1 ? argv[1][0] - '0' : 3; switch(arg) { @@ -836,7 +900,8 @@ def fasta(self, name, double_rep): case 5: n = 19000000*10; break; default: printf("error: %d\\n", arg); return -1; } - ''') + ''', + ) assert 'switch(arg)' in src self.do_benchmark('fasta', src, '') @@ -854,8 +919,7 @@ def test_skinning(self): def test_havlak(self): src = read_file(test_file('havlak.cpp')) # This runs many recursive calls (DFS) and thus needs a larger stack - self.do_benchmark('havlak', src, 'Found', shared_args=['-std=c++11'], - emcc_args=['-sSTACK_SIZE=1MB']) + self.do_benchmark('havlak', src, 'Found', shared_args=['-std=c++11'], emcc_args=['-sSTACK_SIZE=1MB']) def test_base64(self): src = read_file(test_file('base64.c')) @@ -870,111 +934,216 @@ def test_zzz_linpack(self): def output_parser(output): mflops = re.search(r'Unrolled Double Precision ([\d\.]+) Mflops', output).group(1) return 10000.0 / float(mflops) - self.do_benchmark('linpack_double', read_file(test_file('benchmark/linpack2.c')), '''Unrolled Double Precision''', force_c=True, output_parser=output_parser) + + self.do_benchmark( + 'linpack_double', + read_file(test_file('benchmark/linpack2.c')), + '''Unrolled Double Precision''', + force_c=True, + output_parser=output_parser, + ) # Benchmarks the synthetic performance of calling native functions. @non_core def test_native_functions(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('native_functions', read_file(test_file('benchmark/benchmark_ffis.cpp')), 'Total time:', - output_parser=output_parser, - # Not minimal because this uses functions in library_browsers.js - emcc_args=['-sMINIMAL_RUNTIME=0'], - shared_args=['-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'native_functions', + read_file(test_file('benchmark/benchmark_ffis.cpp')), + 'Total time:', + output_parser=output_parser, + # Not minimal because this uses functions in library_browsers.js + emcc_args=['-sMINIMAL_RUNTIME=0'], + shared_args=['-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) # Benchmarks the synthetic performance of calling function pointers. @non_core def test_native_function_pointers(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('native_functions', read_file(test_file('benchmark/benchmark_ffis.cpp')), 'Total time:', - output_parser=output_parser, - # Not minimal because this uses functions in library_browsers.js - emcc_args=['-sMINIMAL_RUNTIME=0'], - shared_args=['-DBENCHMARK_FUNCTION_POINTER=1', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'native_functions', + read_file(test_file('benchmark/benchmark_ffis.cpp')), + 'Total time:', + output_parser=output_parser, + # Not minimal because this uses functions in library_browsers.js + emcc_args=['-sMINIMAL_RUNTIME=0'], + shared_args=['-DBENCHMARK_FUNCTION_POINTER=1', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) # Benchmarks the synthetic performance of calling "foreign" JavaScript functions. @non_core def test_foreign_functions(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('foreign_functions', read_file(test_file('benchmark/benchmark_ffis.cpp')), 'Total time:', - output_parser=output_parser, - # Not minimal because this uses functions in library_browsers.js - emcc_args=['--js-library', test_file('benchmark/benchmark_ffis.js'), '-sMINIMAL_RUNTIME=0'], - shared_args=['-DBENCHMARK_FOREIGN_FUNCTION=1', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'foreign_functions', + read_file(test_file('benchmark/benchmark_ffis.cpp')), + 'Total time:', + output_parser=output_parser, + # Not minimal because this uses functions in library_browsers.js + emcc_args=['--js-library', test_file('benchmark/benchmark_ffis.js'), '-sMINIMAL_RUNTIME=0'], + shared_args=['-DBENCHMARK_FOREIGN_FUNCTION=1', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memcpy_128b(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memcpy_128b', read_file(test_file('benchmark/benchmark_memcpy.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMAX_COPY=128', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memcpy_128b', + read_file(test_file('benchmark/benchmark_memcpy.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMAX_COPY=128', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memcpy_4k(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memcpy_4k', read_file(test_file('benchmark/benchmark_memcpy.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=128', '-DMAX_COPY=4096', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memcpy_4k', + read_file(test_file('benchmark/benchmark_memcpy.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=128', '-DMAX_COPY=4096', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memcpy_16k(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memcpy_16k', read_file(test_file('benchmark/benchmark_memcpy.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=4096', '-DMAX_COPY=16384', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memcpy_16k', + read_file(test_file('benchmark/benchmark_memcpy.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=4096', '-DMAX_COPY=16384', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memcpy_1mb(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memcpy_1mb', read_file(test_file('benchmark/benchmark_memcpy.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=16384', '-DMAX_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memcpy_1mb', + read_file(test_file('benchmark/benchmark_memcpy.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=16384', '-DMAX_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memcpy_16mb(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memcpy_16mb', read_file(test_file('benchmark/benchmark_memcpy.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memcpy_16mb', + read_file(test_file('benchmark/benchmark_memcpy.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memset_128b(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memset_128b', read_file(test_file('benchmark/benchmark_memset.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMAX_COPY=128', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memset_128b', + read_file(test_file('benchmark/benchmark_memset.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMAX_COPY=128', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memset_4k(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memset_4k', read_file(test_file('benchmark/benchmark_memset.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=128', '-DMAX_COPY=4096', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memset_4k', + read_file(test_file('benchmark/benchmark_memset.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=128', '-DMAX_COPY=4096', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memset_16k(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memset_16k', read_file(test_file('benchmark/benchmark_memset.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=4096', '-DMAX_COPY=16384', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memset_16k', + read_file(test_file('benchmark/benchmark_memset.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=4096', '-DMAX_COPY=16384', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memset_1mb(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memset_1mb', read_file(test_file('benchmark/benchmark_memset.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=16384', '-DMAX_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memset_1mb', + read_file(test_file('benchmark/benchmark_memset.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=16384', '-DMAX_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) @non_core def test_memset_16mb(self): def output_parser(output): return float(re.search(r'Total time: ([\d\.]+)', output).group(1)) - self.do_benchmark('memset_16mb', read_file(test_file('benchmark/benchmark_memset.cpp')), 'Total time:', output_parser=output_parser, shared_args=['-DMIN_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')]) + + self.do_benchmark( + 'memset_16mb', + read_file(test_file('benchmark/benchmark_memset.cpp')), + 'Total time:', + output_parser=output_parser, + shared_args=['-DMIN_COPY=1048576', '-DBUILD_FOR_SHELL', '-I' + test_file('benchmark')], + ) def test_malloc_multithreading(self): # Multithreaded malloc test. For emcc we use mimalloc here. src = read_file(test_file('other/test_malloc_multithreading.cpp')) # TODO measure with different numbers of cores and not fixed 4 - self.do_benchmark('malloc_multithreading', src, 'Done.', shared_args=['-DWORKERS=4', '-pthread'], emcc_args=['-sEXIT_RUNTIME', '-sMALLOC=mimalloc']) + self.do_benchmark( + 'malloc_multithreading', + src, + 'Done.', + shared_args=['-DWORKERS=4', '-pthread'], + emcc_args=['-sEXIT_RUNTIME', '-sMALLOC=mimalloc'], + ) def test_matrix_multiply(self): def output_parser(output): return float(re.search(r'Total elapsed: ([\d\.]+)', output).group(1)) - self.do_benchmark('matrix_multiply', read_file(test_file('matrix_multiply.cpp')), 'Total elapsed:', output_parser=output_parser, shared_args=['-I' + test_file('benchmark')]) + + self.do_benchmark( + 'matrix_multiply', + read_file(test_file('matrix_multiply.cpp')), + 'Total elapsed:', + output_parser=output_parser, + shared_args=['-I' + test_file('benchmark')], + ) def lua(self, benchmark, expected, output_parser=None, args_processor=None): self.emcc_args.remove('-Werror') @@ -983,13 +1152,34 @@ def lua(self, benchmark, expected, output_parser=None, args_processor=None): def lib_builder(name, native, env_init): # We force recomputation for the native benchmarker because this benchmark # uses native_exec=True, so we need to copy the native executable - return self.get_library(os.path.join('third_party', 'lua_native' if native else 'lua'), [os.path.join('src', 'lua.o'), os.path.join('src', 'liblua.a')], make=['make', 'generic'], configure=None, native=native, cache_name_extra=name, env_init=env_init, force_rebuild=native) - - self.do_benchmark('lua_' + benchmark, '', expected, - force_c=True, args=[benchmark + '.lua', DEFAULT_ARG], - emcc_args=['--embed-file', benchmark + '.lua', '-sFORCE_FILESYSTEM', '-sMINIMAL_RUNTIME=0'], # not minimal because of files - lib_builder=lib_builder, native_exec=os.path.join('building', 'third_party', 'lua_native', 'src', 'lua'), - output_parser=output_parser, args_processor=args_processor) + return self.get_library( + os.path.join('third_party', 'lua_native' if native else 'lua'), + [os.path.join('src', 'lua.o'), os.path.join('src', 'liblua.a')], + make=['make', 'generic'], + configure=None, + native=native, + cache_name_extra=name, + env_init=env_init, + force_rebuild=native, + ) + + self.do_benchmark( + 'lua_' + benchmark, + '', + expected, + force_c=True, + args=[benchmark + '.lua', DEFAULT_ARG], + emcc_args=[ + '--embed-file', + benchmark + '.lua', + '-sFORCE_FILESYSTEM', + '-sMINIMAL_RUNTIME=0', + ], # not minimal because of files + lib_builder=lib_builder, + native_exec=os.path.join('building', 'third_party', 'lua_native', 'src', 'lua'), + output_parser=output_parser, + args_processor=args_processor, + ) def test_zzz_lua_scimark(self): def output_parser(output): @@ -1006,30 +1196,62 @@ def test_zzz_zlib(self): src = read_file(test_file('benchmark/test_zlib_benchmark.c')) def lib_builder(name, native, env_init): - return self.get_library(os.path.join('third_party', 'zlib'), os.path.join('libz.a'), make_args=['libz.a'], native=native, cache_name_extra=name, env_init=env_init) - - self.do_benchmark('zlib', src, 'ok.', - force_c=True, shared_args=['-I' + test_file('third_party/zlib')], lib_builder=lib_builder) + return self.get_library( + os.path.join('third_party', 'zlib'), + os.path.join('libz.a'), + make_args=['libz.a'], + native=native, + cache_name_extra=name, + env_init=env_init, + ) + + self.do_benchmark( + 'zlib', src, 'ok.', force_c=True, shared_args=['-I' + test_file('third_party/zlib')], lib_builder=lib_builder + ) def test_zzz_coremark(self): src = read_file(test_file('third_party/coremark/core_main.c')) def lib_builder(name, native, env_init): - return self.get_library('third_party/coremark', [os.path.join('coremark.a')], configure=None, native=native, cache_name_extra=name, env_init=env_init) + return self.get_library( + 'third_party/coremark', + [os.path.join('coremark.a')], + configure=None, + native=native, + cache_name_extra=name, + env_init=env_init, + ) def output_parser(output): iters_sec = re.search(r'Iterations/Sec : ([\d\.]+)', output).group(1) return 100000.0 / float(iters_sec) - self.do_benchmark('coremark', src, 'Correct operation validated.', shared_args=['-I' + test_file('third_party/coremark')], lib_builder=lib_builder, output_parser=output_parser, force_c=True) + self.do_benchmark( + 'coremark', + src, + 'Correct operation validated.', + shared_args=['-I' + test_file('third_party/coremark')], + lib_builder=lib_builder, + output_parser=output_parser, + force_c=True, + ) def test_zzz_box2d(self): src = read_file(test_file('benchmark/test_box2d_benchmark.cpp')) def lib_builder(name, native, env_init): - return self.get_library(os.path.join('third_party', 'box2d'), ['box2d.a'], configure=None, native=native, cache_name_extra=name, env_init=env_init) - - self.do_benchmark('box2d', src, 'frame averages', shared_args=['-I' + test_file('third_party/box2d')], lib_builder=lib_builder) + return self.get_library( + os.path.join('third_party', 'box2d'), + ['box2d.a'], + configure=None, + native=native, + cache_name_extra=name, + env_init=env_init, + ) + + self.do_benchmark( + 'box2d', src, 'frame averages', shared_args=['-I' + test_file('third_party/box2d')], lib_builder=lib_builder + ) def test_zzz_bullet(self): self.emcc_args.remove('-Werror') @@ -1038,40 +1260,64 @@ def test_zzz_bullet(self): src += read_file(test_file('third_party/bullet/Demos/Benchmarks/main.cpp')) def lib_builder(name, native, env_init): - return self.get_library(str(Path('third_party/bullet')), - [Path('src/.libs/libBulletDynamics.a'), - Path('src/.libs/libBulletCollision.a'), - Path('src/.libs/libLinearMath.a')], - # The --host parameter is needed for 2 reasons: - # 1) bullet in it's configure.ac tries to do platform detection and will fail on unknown platforms - # 2) configure will try to compile and run a test file to check if the C compiler is sane. As Cheerp - # will generate a wasm file (which cannot be run), configure will fail. Passing `--host` enables - # cross compile mode, which lets configure complete happily. - configure_args=['--disable-demos', '--disable-dependency-tracking', '--host=i686-unknown-linux'], native=native, cache_name_extra=name, env_init=env_init) - - self.do_benchmark('bullet', src, '\nok.\n', - shared_args=['-I' + test_file('third_party/bullet/src'), '-I' + test_file('third_party/bullet/Demos/Benchmarks')], - lib_builder=lib_builder) + return self.get_library( + str(Path('third_party/bullet')), + [ + Path('src/.libs/libBulletDynamics.a'), + Path('src/.libs/libBulletCollision.a'), + Path('src/.libs/libLinearMath.a'), + ], + # The --host parameter is needed for 2 reasons: + # 1) bullet in it's configure.ac tries to do platform detection and will fail on unknown platforms + # 2) configure will try to compile and run a test file to check if the C compiler is sane. As Cheerp + # will generate a wasm file (which cannot be run), configure will fail. Passing `--host` enables + # cross compile mode, which lets configure complete happily. + configure_args=['--disable-demos', '--disable-dependency-tracking', '--host=i686-unknown-linux'], + native=native, + cache_name_extra=name, + env_init=env_init, + ) + + self.do_benchmark( + 'bullet', + src, + '\nok.\n', + shared_args=['-I' + test_file('third_party/bullet/src'), '-I' + test_file('third_party/bullet/Demos/Benchmarks')], + lib_builder=lib_builder, + ) def test_zzz_lzma(self): src = read_file(test_file('benchmark/test_lzma_benchmark.c')) def lib_builder(name, native, env_init): - return self.get_library(os.path.join('third_party', 'lzma'), [os.path.join('lzma.a')], configure=None, native=native, cache_name_extra=name, env_init=env_init) + return self.get_library( + os.path.join('third_party', 'lzma'), + [os.path.join('lzma.a')], + configure=None, + native=native, + cache_name_extra=name, + env_init=env_init, + ) self.do_benchmark('lzma', src, 'ok.', shared_args=['-I' + test_file('third_party/lzma')], lib_builder=lib_builder) def test_zzz_sqlite(self): src = read_file(test_file('third_party/sqlite/sqlite3.c')) + read_file(test_file('sqlite/speedtest1.c')) - self.do_benchmark('sqlite', src, 'TOTAL...', - native_args=['-ldl', '-pthread'], - shared_args=['-I' + test_file('third_party/sqlite')], - # not minimal because of files - emcc_args=['-sFILESYSTEM', '-sMINIMAL_RUNTIME=0'], - force_c=True) + self.do_benchmark( + 'sqlite', + src, + 'TOTAL...', + native_args=['-ldl', '-pthread'], + shared_args=['-I' + test_file('third_party/sqlite')], + # not minimal because of files + emcc_args=['-sFILESYSTEM', '-sMINIMAL_RUNTIME=0'], + force_c=True, + ) def test_zzz_poppler(self): - utils.write_file('pre.js', ''' + utils.write_file( + 'pre.js', + ''' var benchmarkArgument = %s; var benchmarkArgumentToPageCount = { '0': 0, @@ -1107,17 +1353,27 @@ def test_zzz_poppler(self): out(files.length + ' files emitted, total output size: ' + totalSize + ', hashed printout: ' + hash); }; } - ''' % DEFAULT_ARG) + ''' + % DEFAULT_ARG, + ) def lib_builder(name, native, env_init): return self.get_poppler_library(env_init=env_init) # TODO: Fix poppler native build and remove skip_native=True - self.do_benchmark('poppler', '', 'hashed printout', - shared_args=['-I' + test_file('poppler/include'), - '-I' + test_file('freetype/include')], - emcc_args=['-sFILESYSTEM', '--pre-js=pre.js', '--embed-file', - test_file('poppler/emscripten_html5.pdf') + '@input.pdf', - '-sERROR_ON_UNDEFINED_SYMBOLS=0', - '-sMINIMAL_RUNTIME=0'], # not minimal because of files - lib_builder=lib_builder, skip_native=True) + self.do_benchmark( + 'poppler', + '', + 'hashed printout', + shared_args=['-I' + test_file('poppler/include'), '-I' + test_file('freetype/include')], + emcc_args=[ + '-sFILESYSTEM', + '--pre-js=pre.js', + '--embed-file', + test_file('poppler/emscripten_html5.pdf') + '@input.pdf', + '-sERROR_ON_UNDEFINED_SYMBOLS=0', + '-sMINIMAL_RUNTIME=0', + ], # not minimal because of files + lib_builder=lib_builder, + skip_native=True, + ) diff --git a/test/test_interactive.py b/test/test_interactive.py index 511ba785ce831..673b76bc2c20f 100644 --- a/test/test_interactive.py +++ b/test/test_interactive.py @@ -25,7 +25,16 @@ def setUpClass(cls): print() def test_html5_fullscreen(self): - self.btest('test_html5_fullscreen.c', expected='0', args=['-sDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR', '-sEXPORTED_FUNCTIONS=_requestFullscreen,_enterSoftFullscreen,_main', '--shell-file', test_file('test_html5_fullscreen.html')]) + self.btest( + 'test_html5_fullscreen.c', + expected='0', + args=[ + '-sDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR', + '-sEXPORTED_FUNCTIONS=_requestFullscreen,_enterSoftFullscreen,_main', + '--shell-file', + test_file('test_html5_fullscreen.html'), + ], + ) def test_html5_emscripten_exit_with_escape(self): self.btest('test_html5_emscripten_exit_fullscreen.c', expected='1', args=['-DEXIT_WITH_F']) @@ -65,7 +74,21 @@ def test_sdl_audio(self): create_file('bad.ogg', 'I claim to be audio, but am lying') # use closure to check for a possible bug with closure minifying away newer Audio() attributes - self.btest_exit('test_sdl_audio.c', args=['--preload-file', 'sound.ogg', '--preload-file', 'sound2.wav', '--embed-file', 'the_entertainer.ogg', '--preload-file', 'noise.ogg', '--preload-file', 'bad.ogg']) + self.btest_exit( + 'test_sdl_audio.c', + args=[ + '--preload-file', + 'sound.ogg', + '--preload-file', + 'sound2.wav', + '--embed-file', + 'the_entertainer.ogg', + '--preload-file', + 'noise.ogg', + '--preload-file', + 'bad.ogg', + ], + ) # print('SDL2') # check sdl2 as well @@ -76,10 +99,12 @@ def test_sdl_audio(self): # self.run_process([EMCC, '-O1', '--closure', '0', '--minify=0', 'sdl_audio.c', '--preload-file', 'sound.ogg', '--preload-file', 'sound2.wav', '--embed-file', 'the_entertainer.ogg', '--preload-file', 'noise.ogg', '--preload-file', 'bad.ogg', '-o', 'page.html', '-sEXPORTED_FUNCTIONS=[_main,_play,_play2', '-sUSE_SDL=2', '-DUSE_SDL2']).communicate() # self.run_browser('page.html', '', '/report_result?1') - @parameterized({ - '': ([],), - 'wasmfs': (['-sWASMFS'],), - }) + @parameterized( + { + '': ([],), + 'wasmfs': (['-sWASMFS'],), + } + ) def test_sdl_audio_mix_channels(self, args): shutil.copy(test_file('sounds/noise.ogg'), 'sound.ogg') @@ -88,111 +113,241 @@ def test_sdl_audio_mix_channels(self, args): def test_sdl_audio_mix_channels_halt(self): shutil.copy(test_file('sounds/the_entertainer.ogg'), '.') - self.btest_exit('test_sdl_audio_mix_channels_halt.c', args=['-O2', '--minify=0', '--preload-file', 'the_entertainer.ogg']) + self.btest_exit( + 'test_sdl_audio_mix_channels_halt.c', args=['-O2', '--minify=0', '--preload-file', 'the_entertainer.ogg'] + ) def test_sdl_audio_mix_playing(self): shutil.copy(test_file('sounds/noise.ogg'), '.') self.btest_exit('test_sdl_audio_mix_playing.c', args=['-O2', '--minify=0', '--preload-file', 'noise.ogg']) - @parameterized({ - '': ([],), - 'wasmfs': (['-sWASMFS'],), - }) + @parameterized( + { + '': ([],), + 'wasmfs': (['-sWASMFS'],), + } + ) def test_sdl_audio_mix(self, args): shutil.copy(test_file('sounds/pluck.ogg'), 'sound.ogg') shutil.copy(test_file('sounds/the_entertainer.ogg'), 'music.ogg') shutil.copy(test_file('sounds/noise.ogg'), 'noise.ogg') - self.btest_exit('test_sdl_audio_mix.c', args=['-O2', '--minify=0', '--preload-file', 'sound.ogg', '--preload-file', 'music.ogg', '--preload-file', 'noise.ogg'] + args) + self.btest_exit( + 'test_sdl_audio_mix.c', + args=[ + '-O2', + '--minify=0', + '--preload-file', + 'sound.ogg', + '--preload-file', + 'music.ogg', + '--preload-file', + 'noise.ogg', + ] + + args, + ) def test_sdl_audio_panning(self): shutil.copy(test_file('sounds/the_entertainer.wav'), '.') # use closure to check for a possible bug with closure minifying away newer Audio() attributes - self.btest_exit('test_sdl_audio_panning.c', args=['-O2', '--closure=1', '--minify=0', '--preload-file', 'the_entertainer.wav', '-sEXPORTED_FUNCTIONS=_main,_play']) + self.btest_exit( + 'test_sdl_audio_panning.c', + args=[ + '-O2', + '--closure=1', + '--minify=0', + '--preload-file', + 'the_entertainer.wav', + '-sEXPORTED_FUNCTIONS=_main,_play', + ], + ) def test_sdl_audio_beeps(self): # use closure to check for a possible bug with closure minifying away newer Audio() attributes - self.btest_exit('test_sdl_audio_beep.cpp', args=['-O2', '--closure=1', '--minify=0', '-sDISABLE_EXCEPTION_CATCHING=0', '-o', 'page.html']) + self.btest_exit( + 'test_sdl_audio_beep.cpp', + args=['-O2', '--closure=1', '--minify=0', '-sDISABLE_EXCEPTION_CATCHING=0', '-o', 'page.html'], + ) def test_sdl2_mixer_wav(self): shutil.copy(test_file('sounds/the_entertainer.wav'), 'sound.wav') - self.btest_exit('browser/test_sdl2_mixer_wav.c', args=[ - '-O2', - '-sUSE_SDL=2', - '-sUSE_SDL_MIXER=2', - '-sINITIAL_MEMORY=33554432', - '--preload-file', 'sound.wav' - ]) - - @parameterized({ - 'wav': ([], '0', 'the_entertainer.wav'), - 'ogg': (['ogg'], 'MIX_INIT_OGG', 'alarmvictory_1.ogg'), - 'mp3': (['mp3'], 'MIX_INIT_MP3', 'pudinha.mp3'), - }) + self.btest_exit( + 'browser/test_sdl2_mixer_wav.c', + args=['-O2', '-sUSE_SDL=2', '-sUSE_SDL_MIXER=2', '-sINITIAL_MEMORY=33554432', '--preload-file', 'sound.wav'], + ) + + @parameterized( + { + 'wav': ([], '0', 'the_entertainer.wav'), + 'ogg': (['ogg'], 'MIX_INIT_OGG', 'alarmvictory_1.ogg'), + 'mp3': (['mp3'], 'MIX_INIT_MP3', 'pudinha.mp3'), + } + ) def test_sdl2_mixer_music(self, formats, flags, music_name): shutil.copy(test_file('sounds', music_name), '.') - self.btest('browser/test_sdl2_mixer_music.c', expected='exit:0', args=[ - '-O2', - '--minify=0', - '--preload-file', music_name, - '-DSOUND_PATH=' + json.dumps(music_name), - '-DFLAGS=' + flags, - '-sUSE_SDL=2', - '-sUSE_SDL_MIXER=2', - '-sSDL2_MIXER_FORMATS=' + json.dumps(formats), - '-sINITIAL_MEMORY=33554432' - ]) + self.btest( + 'browser/test_sdl2_mixer_music.c', + expected='exit:0', + args=[ + '-O2', + '--minify=0', + '--preload-file', + music_name, + '-DSOUND_PATH=' + json.dumps(music_name), + '-DFLAGS=' + flags, + '-sUSE_SDL=2', + '-sUSE_SDL_MIXER=2', + '-sSDL2_MIXER_FORMATS=' + json.dumps(formats), + '-sINITIAL_MEMORY=33554432', + ], + ) def test_sdl2_audio_beeps(self): # use closure to check for a possible bug with closure minifying away newer Audio() attributes # TODO: investigate why this does not pass - self.btest_exit('browser/test_sdl2_audio_beep.cpp', args=['-O2', '--closure=1', '--minify=0', '-sDISABLE_EXCEPTION_CATCHING=0', '-sUSE_SDL=2']) - - @parameterized({ - '': ([],), - 'proxy_to_pthread': (['-sPROXY_TO_PTHREAD', '-pthread'],), - }) + self.btest_exit( + 'browser/test_sdl2_audio_beep.cpp', + args=['-O2', '--closure=1', '--minify=0', '-sDISABLE_EXCEPTION_CATCHING=0', '-sUSE_SDL=2'], + ) + + @parameterized( + { + '': ([],), + 'proxy_to_pthread': (['-sPROXY_TO_PTHREAD', '-pthread'],), + } + ) def test_openal_playback(self, args): shutil.copy(test_file('sounds/audio.wav'), '.') self.btest('openal/test_openal_playback.c', '1', args=['-O2', '--preload-file', 'audio.wav'] + args) def test_openal_buffers(self): - self.btest_exit('openal/test_openal_buffers.c', args=['--preload-file', test_file('sounds/the_entertainer.wav') + '@/'],) + self.btest_exit( + 'openal/test_openal_buffers.c', + args=['--preload-file', test_file('sounds/the_entertainer.wav') + '@/'], + ) def test_openal_buffers_animated_pitch(self): - self.btest_exit('openal/test_openal_buffers.c', args=['-DTEST_ANIMATED_PITCH=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/'],) + self.btest_exit( + 'openal/test_openal_buffers.c', + args=['-DTEST_ANIMATED_PITCH=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/'], + ) def test_openal_looped_pitched_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=['-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'], + ) def test_openal_looped_seek_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_LOOPED_SEEK_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_LOOPED_SEEK_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_animated_looped_pitched_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ANIMATED_LOOPED_PITCHED_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ANIMATED_LOOPED_PITCHED_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_animated_looped_distance_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ANIMATED_LOOPED_DISTANCE_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ANIMATED_LOOPED_DISTANCE_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_animated_looped_doppler_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ANIMATED_LOOPED_DOPPLER_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ANIMATED_LOOPED_DOPPLER_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_animated_looped_panned_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ANIMATED_LOOPED_PANNED_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ANIMATED_LOOPED_PANNED_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_animated_looped_relative_playback(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ANIMATED_LOOPED_RELATIVE_PLAYBACK=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ANIMATED_LOOPED_RELATIVE_PLAYBACK=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_al_soft_loop_points(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_AL_SOFT_LOOP_POINTS=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_AL_SOFT_LOOP_POINTS=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_alc_soft_pause_device(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_ALC_SOFT_PAUSE_DEVICE=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_ALC_SOFT_PAUSE_DEVICE=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_al_soft_source_spatialize(self): - self.btest('openal/test_openal_playback.c', '1', args=['-DTEST_AL_SOFT_SOURCE_SPATIALIZE=1', '-DTEST_LOOPED_PLAYBACK=1', '--preload-file', test_file('sounds/the_entertainer.wav') + '@/audio.wav'],) + self.btest( + 'openal/test_openal_playback.c', + '1', + args=[ + '-DTEST_AL_SOFT_SOURCE_SPATIALIZE=1', + '-DTEST_LOOPED_PLAYBACK=1', + '--preload-file', + test_file('sounds/the_entertainer.wav') + '@/audio.wav', + ], + ) def test_openal_capture(self): self.btest_exit('openal/test_openal_capture.c') @@ -200,9 +355,18 @@ def test_openal_capture(self): def get_freealut_library(self): self.emcc_args += ['-Wno-pointer-sign'] if WINDOWS and shutil.which('cmake'): - return self.get_library(os.path.join('third_party', 'freealut'), 'libalut.a', configure=['cmake', '.'], configure_args=['-DBUILD_TESTS=ON']) + return self.get_library( + os.path.join('third_party', 'freealut'), + 'libalut.a', + configure=['cmake', '.'], + configure_args=['-DBUILD_TESTS=ON'], + ) else: - return self.get_library(os.path.join('third_party', 'freealut'), os.path.join('src', '.libs', 'libalut.a'), configure_args=['--disable-shared']) + return self.get_library( + os.path.join('third_party', 'freealut'), + os.path.join('src', '.libs', 'libalut.a'), + configure_args=['--disable-shared'], + ) def test_freealut(self): src = test_file('third_party/freealut/examples/hello_world.c') @@ -232,42 +396,58 @@ def test_glut_fullscreen(self): self.btest_exit('glut_fullscreen.c', args=['-lglut', '-lGL']) def test_cpuprofiler_memoryprofiler(self): - self.btest('hello_world_gles.c', expected='0', args=['-DLONGTEST=1', '-DTEST_MEMORYPROFILER_ALLOCATIONS_MAP=1', '-O2', '--cpuprofiler', '--memoryprofiler']) + self.btest( + 'hello_world_gles.c', + expected='0', + args=['-DLONGTEST=1', '-DTEST_MEMORYPROFILER_ALLOCATIONS_MAP=1', '-O2', '--cpuprofiler', '--memoryprofiler'], + ) def test_threadprofiler(self): - args = ['-O2', '--threadprofiler', - '-pthread', - '-sASSERTIONS', - '-DTEST_THREAD_PROFILING=1', - '-sPTHREAD_POOL_SIZE=16', - '-sINITIAL_MEMORY=64mb', - '--shell-file', test_file('pthread/test_pthread_mandelbrot_shell.html')] + args = [ + '-O2', + '--threadprofiler', + '-pthread', + '-sASSERTIONS', + '-DTEST_THREAD_PROFILING=1', + '-sPTHREAD_POOL_SIZE=16', + '-sINITIAL_MEMORY=64mb', + '--shell-file', + test_file('pthread/test_pthread_mandelbrot_shell.html'), + ] self.btest_exit('pthread/test_pthread_mandelbrot.cpp', args=args) # Test that event backproxying works. def test_html5_callbacks_on_calling_thread(self): # TODO: Make this automatic by injecting mouse event in e.g. shell html file. for args in ([], ['-DTEST_SYNC_BLOCKING_LOOP=1']): - self.btest('html5_callbacks_on_calling_thread.c', expected='1', args=args + ['-sDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR', '-pthread', '-sPROXY_TO_PTHREAD']) + self.btest( + 'html5_callbacks_on_calling_thread.c', + expected='1', + args=args + ['-sDISABLE_DEPRECATED_FIND_EVENT_TARGET_BEHAVIOR', '-pthread', '-sPROXY_TO_PTHREAD'], + ) # Test that it is possible to register HTML5 event callbacks on either main browser thread, or # application main thread, and that the application can manually proxy the event from main browser # thread to the application main thread, to implement event suppression capabilities. - @parameterized({ - '': ([],), - 'pthread': (['-pthread'],), - 'proxy_to_pthread': (['-pthread', '-sPROXY_TO_PTHREAD'],), - }) + @parameterized( + { + '': ([],), + 'pthread': (['-pthread'],), + 'proxy_to_pthread': (['-pthread', '-sPROXY_TO_PTHREAD'],), + } + ) def test_html5_event_callback_in_two_threads(self, args): # TODO: Make this automatic by injecting enter key press in e.g. shell html file. self.btest_exit('html5_event_callback_in_two_threads.c', args=args) # Test that emscripten_hide_mouse() is callable from pthreads (and proxies to main # thread to obtain the proper window.devicePixelRatio value). - @parameterized({ - '': ([],), - 'threads': (['-pthread'],), - }) + @parameterized( + { + '': ([],), + 'threads': (['-pthread'],), + } + ) def test_emscripten_hide_mouse(self, args): self.btest('emscripten_hide_mouse.c', expected='0', args=args) @@ -275,12 +455,16 @@ def test_emscripten_hide_mouse(self, args): # thread has exited). The intent of this is to stress graceful deinit semantics, so that it is not # possible to "taint" a Canvas to a bad state after a rendering thread in a program quits and # restarts. (perhaps e.g. between level loads, or subsystem loads/restarts or something like that) - @parameterized({ - '': (['-sOFFSCREENCANVAS_SUPPORT', '-DTEST_OFFSCREENCANVAS=1'],), - 'ofb': (['-sOFFSCREEN_FRAMEBUFFER'],), - }) + @parameterized( + { + '': (['-sOFFSCREENCANVAS_SUPPORT', '-DTEST_OFFSCREENCANVAS=1'],), + 'ofb': (['-sOFFSCREEN_FRAMEBUFFER'],), + } + ) def test_webgl_offscreen_canvas_in_two_pthreads(self, args): - self.btest('gl_in_two_pthreads.c', expected='1', args=args + ['-pthread', '-lGL', '-sGL_DEBUG', '-sPROXY_TO_PTHREAD']) + self.btest( + 'gl_in_two_pthreads.c', expected='1', args=args + ['-pthread', '-lGL', '-sGL_DEBUG', '-sPROXY_TO_PTHREAD'] + ) # Tests creating a Web Audio context using Emscripten library_webaudio.js feature. @also_with_minimal_runtime @@ -290,13 +474,21 @@ def test_web_audio(self): # Tests simple AudioWorklet noise generation @also_with_minimal_runtime def test_audio_worklet(self): - self.btest('webaudio/audioworklet.c', expected='0', args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '--preload-file', test_file('hello_world.c') + '@/']) + self.btest( + 'webaudio/audioworklet.c', + expected='0', + args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '--preload-file', test_file('hello_world.c') + '@/'], + ) self.btest('webaudio/audioworklet.c', expected='0', args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-pthread']) # Tests AudioWorklet with emscripten_futex_wake(). @also_with_minimal_runtime def test_audio_worklet_emscripten_futex_wake(self): - self.btest('webaudio/audioworklet_emscripten_futex_wake.cpp', expected='0', args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-pthread', '-sPTHREAD_POOL_SIZE=2']) + self.btest( + 'webaudio/audioworklet_emscripten_futex_wake.cpp', + expected='0', + args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-pthread', '-sPTHREAD_POOL_SIZE=2'], + ) # Tests a second AudioWorklet example: sine wave tone generator. def test_audio_worklet_tone_generator(self): @@ -304,7 +496,11 @@ def test_audio_worklet_tone_generator(self): # Tests that AUDIO_WORKLET+MINIMAL_RUNTIME+MODULARIZE combination works together. def test_audio_worklet_modularize(self): - self.btest('webaudio/audioworklet.c', expected='0', args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-sMINIMAL_RUNTIME', '-sMODULARIZE']) + self.btest( + 'webaudio/audioworklet.c', + expected='0', + args=['-sAUDIO_WORKLET', '-sWASM_WORKERS', '-sMINIMAL_RUNTIME', '-sMODULARIZE'], + ) class interactive64(interactive): diff --git a/test/test_posixtest.py b/test/test_posixtest.py index 506ac006ae135..9de36916327a4 100644 --- a/test/test_posixtest.py +++ b/test/test_posixtest.py @@ -24,16 +24,12 @@ class posixtest(RunnerCore): This class get populated dynamically below. """ + pass def filter_tests(all_tests): - prefixes = [ - 'pthread_', - 'strftime', - 'asctime', - 'gmtime' - ] + prefixes = ['pthread_', 'strftime', 'asctime', 'gmtime'] def enable_test(t): return any(t.startswith(p) for p in prefixes) @@ -155,20 +151,21 @@ def get_tests(): def make_test(name, testfile, browser): - @node_pthreads def f(self): if name in disabled: self.skipTest(disabled[name]) - args = ['-I' + os.path.join(testsuite_root, 'include'), - '-Werror', - '-Wno-format-security', - '-Wno-int-conversion', - '-Wno-format', - '-pthread', - '-sEXIT_RUNTIME', - '-sTOTAL_MEMORY=256mb', - '-sPTHREAD_POOL_SIZE=40'] + args = [ + '-I' + os.path.join(testsuite_root, 'include'), + '-Werror', + '-Wno-format-security', + '-Wno-int-conversion', + '-Wno-format', + '-pthread', + '-sEXIT_RUNTIME', + '-sTOTAL_MEMORY=256mb', + '-sPTHREAD_POOL_SIZE=40', + ] if browser: self.btest_exit(testfile, args=args) else: diff --git a/test/test_posixtest_browser.py b/test/test_posixtest_browser.py index 98298aa668c9e..3b163f0877274 100644 --- a/test/test_posixtest_browser.py +++ b/test/test_posixtest_browser.py @@ -15,4 +15,5 @@ class posixtest_browser(BrowserCore): This class get populated dynamically below. """ + pass diff --git a/test/test_sanity.py b/test/test_sanity.py index 17b95aac6824a..6d69fd90da5b8 100644 --- a/test/test_sanity.py +++ b/test/test_sanity.py @@ -110,7 +110,10 @@ def setUpClass(cls): print() print('Running sanity checks.') - print('WARNING: This will modify %s, and in theory can break it although it should be restored properly. A backup will be saved in %s_backup' % (EM_CONFIG, EM_CONFIG)) + print( + 'WARNING: This will modify %s, and in theory can break it although it should be restored properly. A backup will be saved in %s_backup' + % (EM_CONFIG, EM_CONFIG) + ) print() print('>>> the original settings file is:') print(utils.read_file(EM_CONFIG).strip()) @@ -167,7 +170,11 @@ def test_aaa_normal(self): def test_firstrun(self): default_config = path_from_root('.emscripten') output = self.do([EMCC, '-v']) - self.assertContained('emcc: warning: config file not found: %s. You can create one by hand or run `emcc --generate-config`' % default_config, output) + self.assertContained( + 'emcc: warning: config file not found: %s. You can create one by hand or run `emcc --generate-config`' + % default_config, + output, + ) try: temp_bin = tempfile.mkdtemp() @@ -222,7 +229,7 @@ def make_new_executable(name): if 'blah' in settings: self.assertContained('error: error in evaluating config file (%s)' % default_config, output) elif 'runner' not in ' '.join(command): - self.assertContained('error: NODE_JS is set to empty value', output) # sanity check should fail + self.assertContained('error: NODE_JS is set to empty value', output) # sanity check should fail finally: delete_file(default_config) @@ -251,7 +258,7 @@ def test_llvm(self): expected_x = real_version_x + inc_x expected_y = real_version_y + inc_y if expected_x < 0 or expected_y < 0: - continue # must be a valid llvm version + continue # must be a valid llvm version print("mod LLVM version: %d %d -> %d %d" % (real_version_x, real_version_y, expected_x, expected_y)) make_fake_clang(self.in_dir('fake', 'clang'), '%s.%s' % (expected_x, expected_y)) make_fake_tool(self.in_dir('fake', 'llvm-ar'), '%s.%s' % (expected_x, expected_y)) @@ -282,21 +289,27 @@ def test_node(self): ensure_dir('fake') - for version, succeed in (('v0.8.0', False), - ('v4.1.0', False), - ('v10.18.0', False), - ('v16.20.0', True), - ('v16.20.1-pre', True), - ('cheez', False)): + for version, succeed in ( + ('v0.8.0', False), + ('v4.1.0', False), + ('v10.18.0', False), + ('v16.20.0', True), + ('v16.20.1-pre', True), + ('cheez', False), + ): print(version, succeed) delete_file(SANITY_FILE) - utils.write_file(self.in_dir('fake', 'nodejs'), '''#!/bin/sh + utils.write_file( + self.in_dir('fake', 'nodejs'), + '''#!/bin/sh if [ $1 = "--version" ]; then echo "%s" else %s $@ fi -''' % (version, ' '.join(config.NODE_JS))) +''' + % (version, ' '.join(config.NODE_JS)), + ) make_executable(self.in_dir('fake', 'nodejs')) if not succeed: if version[0] == 'v': @@ -313,7 +326,7 @@ def test_emcc(self): # emcc should check sanity if no ${EM_CONFIG}_sanity restore_and_set_up() time.sleep(1) - assert not os.path.exists(SANITY_FILE) # restore is just the settings, not the sanity + assert not os.path.exists(SANITY_FILE) # restore is just the settings, not the sanity output = self.check_working(EMCC) self.assertContained(SANITY_MESSAGE, output) # EMCC should have checked sanity successfully @@ -352,13 +365,16 @@ def test_emcc(self): def test_em_config_env_var(self): # emcc should be configurable directly from EM_CONFIG without any config file restore_and_set_up() - create_file('main.cpp', ''' + create_file( + 'main.cpp', + ''' #include int main() { printf("hello from emcc with no config file\\n"); return 0; } - ''') + ''', + ) wipe() with env_modify({'EM_CONFIG': get_basic_config()}): @@ -438,13 +454,16 @@ def test_FROZEN_CACHE(self): def test_emcc_multiprocess_cache_access(self): restore_and_set_up() - create_file('test.c', r''' + create_file( + 'test.c', + r''' #include int main() { printf("hello, world!\n"); return 0; } - ''') + ''', + ) cache_dir_name = self.in_dir('test_cache') libname = cache.get_lib_name('libc.a') with env_modify({'EM_CACHE': cache_dir_name}): @@ -465,11 +484,7 @@ def test_emcc_multiprocess_cache_access(self): # Exactly one child process should have triggered libc build! self.assertEqual(num_times_libc_was_built, 1) - @parameterized({ - '': [False, False], - 'response_files': [True, False], - 'relative': [False, True] - }) + @parameterized({'': [False, False], 'response_files': [True, False], 'relative': [False, True]}) def test_emcc_cache_flag(self, use_response_files, relative): restore_and_set_up() @@ -478,13 +493,16 @@ def test_emcc_cache_flag(self, use_response_files, relative): else: cache_dir_name = self.in_dir('emscripten_cache') self.assertFalse(os.path.exists(cache_dir_name)) - create_file('test.c', r''' + create_file( + 'test.c', + r''' #include int main() { printf("hello, world!\n"); return 0; } - ''') + ''', + ) args = ['--cache', cache_dir_name] if use_response_files: rsp = response_file.create_response_file(args, shared.TEMP_DIR) @@ -589,11 +607,13 @@ def test_js_engine_path(self): test_path = self.in_dir('fake', 'abcd8765') ensure_dir(test_path) - jsengines = [('d8', config.V8_ENGINE), - ('d8_g', config.V8_ENGINE), - ('js', config.SPIDERMONKEY_ENGINE), - ('node', config.NODE_JS), - ('nodejs', config.NODE_JS)] + jsengines = [ + ('d8', config.V8_ENGINE), + ('d8_g', config.V8_ENGINE), + ('js', config.SPIDERMONKEY_ENGINE), + ('node', config.NODE_JS), + ('nodejs', config.NODE_JS), + ] for filename, engine in jsengines: delete_file(SANITY_FILE) if type(engine) is list: @@ -656,7 +676,9 @@ def test_with_fake(report, expected): with env_modify({'EMCC_DEBUG': '1'}): self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], expected) - test_with_fake('got js backend! JavaScript (asm.js, emscripten) backend', 'LLVM has not been built with the WebAssembly backend') + test_with_fake( + 'got js backend! JavaScript (asm.js, emscripten) backend', 'LLVM has not been built with the WebAssembly backend' + ) delete_dir(shared.CANONICAL_TEMP_DIR) def test_llvm_add_version(self): @@ -684,7 +706,10 @@ def test_required_config_settings(self): self.check_working([EMCC, test_file('hello_world.c')], 'BINARYEN_ROOT is set to empty value in %s' % EM_CONFIG) open(EM_CONFIG, 'a').write('\ndel BINARYEN_ROOT\n') - self.check_working([EMCC, test_file('hello_world.c')], 'BINARYEN_ROOT not set in config (%s), and `wasm-opt` not found in PATH' % EM_CONFIG) + self.check_working( + [EMCC, test_file('hello_world.c')], + 'BINARYEN_ROOT not set in config (%s), and `wasm-opt` not found in PATH' % EM_CONFIG, + ) def test_empty_config(self): restore_and_set_up() @@ -753,7 +778,10 @@ def test_embuilder_with_use_port_syntax(self): restore_and_set_up() self.run_process([EMBUILDER, 'build', 'sdl2_image:formats=png,jpg', '--force']) self.assertExists(os.path.join(config.CACHE, 'sysroot', 'lib', 'wasm32-emscripten', 'libSDL2_image-jpg-png.a')) - self.assertContained('error building port `sdl2_image:formats=invalid` | invalid is not a supported format', self.do([EMBUILDER, 'build', 'sdl2_image:formats=invalid', '--force'])) + self.assertContained( + 'error building port `sdl2_image:formats=invalid` | invalid is not a supported format', + self.do([EMBUILDER, 'build', 'sdl2_image:formats=invalid', '--force']), + ) def test_embuilder_external_ports(self): restore_and_set_up() @@ -777,7 +805,10 @@ def test_binaryen_version(self): f.write('\nBINARYEN_ROOT = "' + self.in_dir('fake') + '"') make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), 'foo') - self.check_working([EMCC, test_file('hello_world.c')], 'error parsing binaryen version (wasm-opt version foo). Please check your binaryen installation') + self.check_working( + [EMCC, test_file('hello_world.c')], + 'error parsing binaryen version (wasm-opt version foo). Please check your binaryen installation', + ) make_fake_tool(self.in_dir('fake', 'bin', 'wasm-opt'), '70') self.check_working([EMCC, test_file('hello_world.c')], 'unexpected binaryen version: 70 (expected ') @@ -789,7 +820,9 @@ def test_bootstrap(self): # Touching package.json should cause compiler to fail with bootstrap message Path(utils.path_from_root('package.json')).touch() err = self.expect_fail([EMCC, test_file('hello_world.c')]) - self.assertContained('emcc: error: emscripten setup is not complete ("npm packages" is out-of-date). Run `bootstrap` to update', err) + self.assertContained( + 'emcc: error: emscripten setup is not complete ("npm packages" is out-of-date). Run `bootstrap` to update', err + ) # Running bootstrap.py should fix that bootstrap = shared.bat_suffix(shared.path_from_root('bootstrap')) diff --git a/test/test_sockets.py b/test/test_sockets.py index 53081075a6e28..6876d8993b5bf 100644 --- a/test/test_sockets.py +++ b/test/test_sockets.py @@ -30,18 +30,18 @@ def clean_processes(processes): if getattr(p, 'exitcode', None) is None and getattr(p, 'returncode', None) is None: # ask nicely (to try and catch the children) try: - p.terminate() # SIGTERM + p.terminate() # SIGTERM except OSError: pass time.sleep(1) # send a forcible kill immediately afterwards. If the process did not die before, this should clean it. try: - p.terminate() # SIGKILL + p.terminate() # SIGKILL except OSError: pass -class WebsockifyServerHarness(): +class WebsockifyServerHarness: def __init__(self, filename, args, listen_port, do_server_check=True): self.processes = [] self.filename = filename @@ -55,7 +55,11 @@ def __enter__(self): # NOTE empty filename support is a hack to support # the current test_enet if self.filename: - cmd = [CLANG_CC, test_file(self.filename), '-o', 'server', '-DSOCKK=%d' % self.target_port] + clang_native.get_clang_native_args() + self.args + cmd = ( + [CLANG_CC, test_file(self.filename), '-o', 'server', '-DSOCKK=%d' % self.target_port] + + clang_native.get_clang_native_args() + + self.args + ) print(cmd) run_process(cmd, env=clang_native.get_clang_native_env()) process = Popen([os.path.abspath('server')]) @@ -68,7 +72,14 @@ def __enter__(self): # source_is_ipv6=True here signals to websockify that it should prefer ipv6 address when # resolving host names. This matches what the node `ws` module does and means that `localhost` # resolves to `::1` on IPv6 systems. - wsp = websockify.WebSocketProxy(verbose=True, source_is_ipv6=True, listen_port=self.listen_port, target_host="127.0.0.1", target_port=self.target_port, run_once=True) + wsp = websockify.WebSocketProxy( + verbose=True, + source_is_ipv6=True, + listen_port=self.listen_port, + target_host="127.0.0.1", + target_port=self.target_port, + run_once=True, + ) self.websockify = multiprocessing.Process(target=wsp.start_server) self.websockify.start() self.processes.append(self.websockify) @@ -76,8 +87,8 @@ def __enter__(self): for _ in range(10): try: if self.do_server_check: - server_sock = socket.create_connection(('localhost', self.target_port), timeout=1) - server_sock.close() + server_sock = socket.create_connection(('localhost', self.target_port), timeout=1) + server_sock.close() proxy_sock = socket.create_connection(('localhost', self.listen_port), timeout=1) proxy_sock.close() break @@ -100,7 +111,7 @@ def __exit__(self, *args, **kwargs): clean_processes(self.processes) -class CompiledServerHarness(): +class CompiledServerHarness: def __init__(self, filename, args, listen_port): self.processes = [] self.filename = filename @@ -117,7 +128,9 @@ def __enter__(self): npm_checked = True # compile the server - proc = run_process([EMCC, '-Werror', test_file(self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args) + proc = run_process( + [EMCC, '-Werror', test_file(self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args + ) print('Socket server build: out:', proc.stdout or '', '/ err:', proc.stderr or '') process = Popen(config.NODE_JS + ['server.js']) @@ -133,7 +146,7 @@ def __exit__(self, *args, **kwargs): # Executes a native executable server process -class BackgroundServerProcess(): +class BackgroundServerProcess: def __init__(self, args): self.processes = [] self.args = args @@ -177,13 +190,15 @@ def setUpClass(cls): # WebsockifyServerHarness uses two port numbers, x and x-1, so increment it by two. # CompiledServerHarness only uses one. Start with 49160 & 49159 as the first server port # addresses. If adding new tests, increment the used port addresses below. - @parameterized({ - 'websockify': [WebsockifyServerHarness, 49160, ['-DTEST_DGRAM=0']], - 'tcp': [CompiledServerHarness, 49161, ['-DTEST_DGRAM=0']], - 'udp': [CompiledServerHarness, 49162, ['-DTEST_DGRAM=1']], - # The following forces non-NULL addr and addlen parameters for the accept call - 'accept_addr': [CompiledServerHarness, 49163, ['-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1']], - }) + @parameterized( + { + 'websockify': [WebsockifyServerHarness, 49160, ['-DTEST_DGRAM=0']], + 'tcp': [CompiledServerHarness, 49161, ['-DTEST_DGRAM=0']], + 'udp': [CompiledServerHarness, 49162, ['-DTEST_DGRAM=1']], + # The following forces non-NULL addr and addlen parameters for the accept call + 'accept_addr': [CompiledServerHarness, 49163, ['-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1']], + } + ) def test_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') @@ -193,19 +208,26 @@ def test_sockets_echo(self, harness_class, port, args): def test_sockets_echo_pthreads(self): with CompiledServerHarness(test_file('sockets/test_sockets_echo_server.c'), [], 49161) as harness: - self.btest_exit('sockets/test_sockets_echo_client.c', args=['-pthread', '-sPROXY_TO_PTHREAD', '-DSOCKK=%d' % harness.listen_port]) + self.btest_exit( + 'sockets/test_sockets_echo_client.c', + args=['-pthread', '-sPROXY_TO_PTHREAD', '-DSOCKK=%d' % harness.listen_port], + ) def test_sdl2_sockets_echo(self): with CompiledServerHarness('sockets/sdl2_net_server.c', ['-sUSE_SDL=2', '-sUSE_SDL_NET=2'], 49164) as harness: - self.btest_exit('sockets/sdl2_net_client.c', args=['-sUSE_SDL=2', '-sUSE_SDL_NET=2', '-DSOCKK=%d' % harness.listen_port]) - - @parameterized({ - 'websockify': [WebsockifyServerHarness, 49166, ['-DTEST_DGRAM=0']], - 'tcp': [CompiledServerHarness, 49167, ['-DTEST_DGRAM=0']], - 'udp': [CompiledServerHarness, 49168, ['-DTEST_DGRAM=1']], - # The following forces non-NULL addr and addlen parameters for the accept call - 'accept_addr': [CompiledServerHarness, 49169, ['-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1']], - }) + self.btest_exit( + 'sockets/sdl2_net_client.c', args=['-sUSE_SDL=2', '-sUSE_SDL_NET=2', '-DSOCKK=%d' % harness.listen_port] + ) + + @parameterized( + { + 'websockify': [WebsockifyServerHarness, 49166, ['-DTEST_DGRAM=0']], + 'tcp': [CompiledServerHarness, 49167, ['-DTEST_DGRAM=0']], + 'udp': [CompiledServerHarness, 49168, ['-DTEST_DGRAM=1']], + # The following forces non-NULL addr and addlen parameters for the accept call + 'accept_addr': [CompiledServerHarness, 49169, ['-DTEST_DGRAM=0', '-DTEST_ACCEPT_ADDR=1']], + } + ) def test_sockets_async_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') @@ -219,11 +241,13 @@ def test_sockets_async_bad_port(self): # getsockopt self.btest_exit('sockets/test_sockets_echo_client.c', args=['-DSOCKK=49169', '-DTEST_ASYNC=1']) - @parameterized({ - 'websockify': [WebsockifyServerHarness, 49171, ['-DTEST_DGRAM=0']], - 'tcp': [CompiledServerHarness, 49172, ['-DTEST_DGRAM=0']], - 'udp': [CompiledServerHarness, 49173, ['-DTEST_DGRAM=1']], - }) + @parameterized( + { + 'websockify': [WebsockifyServerHarness, 49171, ['-DTEST_DGRAM=0']], + 'tcp': [CompiledServerHarness, 49172, ['-DTEST_DGRAM=0']], + 'udp': [CompiledServerHarness, 49173, ['-DTEST_DGRAM=1']], + } + ) def test_sockets_echo_bigdata(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') @@ -236,7 +260,9 @@ def test_sockets_echo_bigdata(self, harness_class, port, args): # re-write the client test with this literal (it's too big to pass via command line) src = read_file(test_file('sockets/test_sockets_echo_client.c')) - create_file('test_sockets_echo_bigdata.c', src.replace('#define MESSAGE "pingtothepong"', '#define MESSAGE "%s"' % message)) + create_file( + 'test_sockets_echo_bigdata.c', src.replace('#define MESSAGE "pingtothepong"', '#define MESSAGE "%s"' % message) + ) with harness_class(test_file('sockets/test_sockets_echo_server.c'), args, port) as harness: self.btest_exit('test_sockets_echo_bigdata.c', args=[sockets_include, '-DSOCKK=%d' % harness.listen_port] + args) @@ -245,16 +271,20 @@ def test_sockets_echo_bigdata(self, harness_class, port, args): def test_sockets_partial(self): for harness in [ WebsockifyServerHarness(test_file('sockets/test_sockets_partial_server.c'), [], 49180), - CompiledServerHarness(test_file('sockets/test_sockets_partial_server.c'), [], 49181) + CompiledServerHarness(test_file('sockets/test_sockets_partial_server.c'), [], 49181), ]: with harness: - self.btest_exit('sockets/test_sockets_partial_client.c', assert_returncode=165, args=['-DSOCKK=%d' % harness.listen_port]) + self.btest_exit( + 'sockets/test_sockets_partial_client.c', assert_returncode=165, args=['-DSOCKK=%d' % harness.listen_port] + ) @no_windows('This test is Unix-specific.') def test_sockets_select_server_down(self): for harness in [ - WebsockifyServerHarness(test_file('sockets/test_sockets_select_server_down_server.c'), [], 49190, do_server_check=False), - CompiledServerHarness(test_file('sockets/test_sockets_select_server_down_server.c'), [], 49191) + WebsockifyServerHarness( + test_file('sockets/test_sockets_select_server_down_server.c'), [], 49190, do_server_check=False + ), + CompiledServerHarness(test_file('sockets/test_sockets_select_server_down_server.c'), [], 49191), ]: with harness: self.btest_exit('sockets/test_sockets_select_server_down_client.c', args=['-DSOCKK=%d' % harness.listen_port]) @@ -263,10 +293,12 @@ def test_sockets_select_server_down(self): def test_sockets_select_server_closes_connection_rw(self): for harness in [ WebsockifyServerHarness(test_file('sockets/test_sockets_echo_server.c'), ['-DCLOSE_CLIENT_AFTER_ECHO'], 49200), - CompiledServerHarness(test_file('sockets/test_sockets_echo_server.c'), ['-DCLOSE_CLIENT_AFTER_ECHO'], 49201) + CompiledServerHarness(test_file('sockets/test_sockets_echo_server.c'), ['-DCLOSE_CLIENT_AFTER_ECHO'], 49201), ]: with harness: - self.btest_exit('sockets/test_sockets_select_server_closes_connection_client_rw.c', args=['-DSOCKK=%d' % harness.listen_port]) + self.btest_exit( + 'sockets/test_sockets_select_server_closes_connection_client_rw.c', args=['-DSOCKK=%d' % harness.listen_port] + ) @no_windows('This test uses Unix-specific build architecture.') def test_enet(self): @@ -281,12 +313,14 @@ def test_enet(self): self.btest_exit('sockets/test_enet_client.c', args=enet + ['-DSOCKK=%d' % harness.listen_port]) @crossplatform - @parameterized({ - 'native': [WebsockifyServerHarness, 59160, ['-DTEST_DGRAM=0']], - 'tcp': [CompiledServerHarness, 59162, ['-DTEST_DGRAM=0']], - 'udp': [CompiledServerHarness, 59164, ['-DTEST_DGRAM=1']], - 'pthread': [CompiledServerHarness, 59166, ['-pthread', '-sPROXY_TO_PTHREAD']], - }) + @parameterized( + { + 'native': [WebsockifyServerHarness, 59160, ['-DTEST_DGRAM=0']], + 'tcp': [CompiledServerHarness, 59162, ['-DTEST_DGRAM=0']], + 'udp': [CompiledServerHarness, 59164, ['-DTEST_DGRAM=1']], + 'pthread': [CompiledServerHarness, 59166, ['-pthread', '-sPROXY_TO_PTHREAD']], + } + ) def test_nodejs_sockets_echo(self, harness_class, port, args): if harness_class == WebsockifyServerHarness and common.EMTEST_LACKS_NATIVE_CLANG: self.skipTest('requires native clang') @@ -294,10 +328,17 @@ def test_nodejs_sockets_echo(self, harness_class, port, args): # Basic test of node client against both a Websockified and compiled echo server. with harness_class(test_file('sockets/test_sockets_echo_server.c'), args, port) as harness: expected = 'do_msg_read: read 14 bytes' - self.do_runf('sockets/test_sockets_echo_client.c', expected, emcc_args=['-DSOCKK=%d' % harness.listen_port] + args) + self.do_runf( + 'sockets/test_sockets_echo_client.c', expected, emcc_args=['-DSOCKK=%d' % harness.listen_port] + args + ) def test_nodejs_sockets_connect_failure(self): - self.do_runf('sockets/test_sockets_echo_client.c', 'connect failed: Connection refused', emcc_args=['-DSOCKK=666'], assert_returncode=NON_ZERO) + self.do_runf( + 'sockets/test_sockets_echo_client.c', + 'connect failed: Connection refused', + emcc_args=['-DSOCKK=666'], + assert_returncode=NON_ZERO, + ) @requires_native_clang def test_nodejs_sockets_echo_subprotocol(self): @@ -305,27 +346,54 @@ def test_nodejs_sockets_echo_subprotocol(self): # server because as long as the subprotocol list contains binary it will configure itself to accept binary # This test also checks that the connect url contains the correct subprotocols. with WebsockifyServerHarness(test_file('sockets/test_sockets_echo_server.c'), [], 59168): - self.run_process([EMCC, '-Werror', test_file('sockets/test_sockets_echo_client.c'), '-o', 'client.js', '-sSOCKET_DEBUG', '-sWEBSOCKET_SUBPROTOCOL="base64, binary"', '-DSOCKK=59168']) + self.run_process( + [ + EMCC, + '-Werror', + test_file('sockets/test_sockets_echo_client.c'), + '-o', + 'client.js', + '-sSOCKET_DEBUG', + '-sWEBSOCKET_SUBPROTOCOL="base64, binary"', + '-DSOCKK=59168', + ] + ) out = self.run_js('client.js') self.assertContained('do_msg_read: read 14 bytes', out) - self.assertContained(['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out) + self.assertContained( + ['connect: ws://127.0.0.1:59168, base64,binary', 'connect: ws://127.0.0.1:59168/, base64,binary'], out + ) @requires_native_clang def test_nodejs_sockets_echo_subprotocol_runtime(self): # Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol. # In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so # the connection would fail without us specifying a valid WebSocket URL in the configuration. - create_file('websocket_pre.js', ''' + create_file( + 'websocket_pre.js', + ''' var Module = { websocket: { url: 'ws://localhost:59168/testA/testB', subprotocol: 'text, base64, binary', } }; - ''') + ''', + ) with WebsockifyServerHarness(test_file('sockets/test_sockets_echo_server.c'), [], 59168): - self.run_process([EMCC, '-Werror', test_file('sockets/test_sockets_echo_client.c'), '-o', 'client.js', '--pre-js=websocket_pre.js', '-sSOCKET_DEBUG', '-DSOCKK=12345']) + self.run_process( + [ + EMCC, + '-Werror', + test_file('sockets/test_sockets_echo_client.c'), + '-o', + 'client.js', + '--pre-js=websocket_pre.js', + '-sSOCKET_DEBUG', + '-DSOCKK=12345', + ] + ) out = self.run_js('client.js') self.assertContained('do_msg_read: read 14 bytes', out) @@ -334,13 +402,17 @@ def test_nodejs_sockets_echo_subprotocol_runtime(self): # Test Emscripten WebSockets API to send and receive text and binary messages against an echo server. # N.B. running this test requires 'npm install ws' in Emscripten root directory # NOTE: Shared buffer is not allowed for websocket sending. - @parameterized({ - '': [[]], - 'shared': [['-sSHARED_MEMORY']], - }) + @parameterized( + { + '': [[]], + 'shared': [['-sSHARED_MEMORY']], + } + ) def test_websocket_send(self, args): with NodeJsWebSocketEchoServerProcess(): - self.btest_exit('websocket/test_websocket_send.c', args=['-lwebsocket', '-sNO_EXIT_RUNTIME', '-sWEBSOCKET_DEBUG'] + args) + self.btest_exit( + 'websocket/test_websocket_send.c', args=['-lwebsocket', '-sNO_EXIT_RUNTIME', '-sWEBSOCKET_DEBUG'] + args + ) # Test that native POSIX sockets API can be used by proxying calls to an intermediate WebSockets # -> POSIX sockets bridge server @@ -348,7 +420,9 @@ def test_posix_proxy_sockets(self): # Build the websocket bridge server self.run_process(['cmake', path_from_root('tools/websocket_to_posix_proxy')]) self.run_process(['cmake', '--build', '.']) - if os.name == 'nt': # This is not quite exact, instead of "isWindows()" this should be "If CMake defaults to building with Visual Studio", but there is no good check for that, so assume Windows==VS. + if ( + os.name == 'nt' + ): # This is not quite exact, instead of "isWindows()" this should be "If CMake defaults to building with Visual Studio", but there is no good check for that, so assume Windows==VS. proxy_server = self.in_dir('Debug', 'websocket_to_posix_proxy.exe') else: proxy_server = self.in_dir('websocket_to_posix_proxy') @@ -356,7 +430,9 @@ def test_posix_proxy_sockets(self): with BackgroundServerProcess([proxy_server, '8080']): with PythonTcpEchoServerProcess('7777'): # Build and run the TCP echo client program with Emscripten - self.btest_exit('websocket/tcp_echo_client.c', args=['-lwebsocket', '-sPROXY_POSIX_SOCKETS', '-pthread', '-sPROXY_TO_PTHREAD']) + self.btest_exit( + 'websocket/tcp_echo_client.c', args=['-lwebsocket', '-sPROXY_POSIX_SOCKETS', '-pthread', '-sPROXY_TO_PTHREAD'] + ) # Test that calling send() right after a socket connect() works. def test_sockets_send_while_connecting(self): diff --git a/tools/building.py b/tools/building.py index c1caf75ca6840..07af4f5d06e07 100644 --- a/tools/building.py +++ b/tools/building.py @@ -60,14 +60,18 @@ def get_building_env(): env['EMSCRIPTEN_TOOLS'] = path_from_root('tools') env['HOST_CC'] = CLANG_CC env['HOST_CXX'] = CLANG_CXX - env['HOST_CFLAGS'] = '-W' # if set to nothing, CFLAGS is used, which we don't want - env['HOST_CXXFLAGS'] = '-W' # if set to nothing, CXXFLAGS is used, which we don't want - env['PKG_CONFIG_LIBDIR'] = cache.get_sysroot_dir('local/lib/pkgconfig') + os.path.pathsep + cache.get_sysroot_dir('lib/pkgconfig') + env['HOST_CFLAGS'] = '-W' # if set to nothing, CFLAGS is used, which we don't want + env['HOST_CXXFLAGS'] = '-W' # if set to nothing, CXXFLAGS is used, which we don't want + env['PKG_CONFIG_LIBDIR'] = ( + cache.get_sysroot_dir('local/lib/pkgconfig') + os.path.pathsep + cache.get_sysroot_dir('lib/pkgconfig') + ) env['PKG_CONFIG_PATH'] = os.environ.get('EM_PKG_CONFIG_PATH', '') env['EMSCRIPTEN'] = path_from_root() env['PATH'] = cache.get_sysroot_dir('bin') + os.pathsep + env['PATH'] env['ACLOCAL_PATH'] = cache.get_sysroot_dir('share/aclocal') - env['CROSS_COMPILE'] = path_from_root('em') # produces /path/to/emscripten/em , which then can have 'cc', 'ar', etc appended to it + env['CROSS_COMPILE'] = path_from_root( + 'em' + ) # produces /path/to/emscripten/em , which then can have 'cc', 'ar', etc appended to it return env @@ -157,9 +161,9 @@ def lld_flags_for_executable(external_symbols): # wasm-ld can strip debug info for us. this strips both the Names # section and DWARF, so we can only use it when we don't need any of # those things. - if settings.DEBUG_LEVEL < 2 and (not settings.EMIT_SYMBOL_MAP and - not settings.EMIT_NAME_SECTION and - not settings.ASYNCIFY): + if settings.DEBUG_LEVEL < 2 and ( + not settings.EMIT_SYMBOL_MAP and not settings.EMIT_NAME_SECTION and not settings.ASYNCIFY + ): cmd.append('--strip-debug') if settings.LINKABLE: @@ -337,6 +341,7 @@ def opt_level_to_str(opt_level, shrink_level=0): def js_optimizer(filename, passes): from . import js_optimizer + try: return js_optimizer.run_on_file(filename, passes) except subprocess.CalledProcessError as e: @@ -394,7 +399,7 @@ def acorn_optimizer(filename, passes, extra_info=None, return_output=False, work # for this, and we are in wasm mode def eval_ctors(js_file, wasm_file, debug_info): if settings.MINIMAL_RUNTIME: - CTOR_ADD_PATTERN = f"wasmExports['{WASM_CALL_CTORS}']();" # TODO test + CTOR_ADD_PATTERN = f"wasmExports['{WASM_CALL_CTORS}']();" # TODO test else: CTOR_ADD_PATTERN = f"addOnInit(wasmExports['{WASM_CALL_CTORS}']);" @@ -503,8 +508,7 @@ def get_closure_compiler_and_env(user_args): def version_split(v): - """Split version setting number (e.g. 162000) into versions string (e.g. "16.2.0") - """ + """Split version setting number (e.g. 162000) into versions string (e.g. "16.2.0")""" v = str(v).rjust(6, '0') assert len(v) == 6 m = re.match(r'(\d{2})(\d{2})(\d{2})', v) @@ -514,10 +518,7 @@ def version_split(v): @ToolchainProfiler.profile() def transpile(filename): - config = { - 'sourceType': 'script', - 'targets': {} - } + config = {'sourceType': 'script', 'targets': {}} if settings.MIN_CHROME_VERSION != UNSUPPORTED: config['targets']['chrome'] = str(settings.MIN_CHROME_VERSION) if settings.MIN_FIREFOX_VERSION != UNSUPPORTED: @@ -533,7 +534,15 @@ def transpile(filename): config_file = shared.get_temp_files().get('babel_config.json').name logger.debug(config_json) utils.write_file(config_file, config_json) - cmd = shared.get_npm_cmd('babel') + [filename, '-o', outfile, '--presets', '@babel/preset-env', '--config-file', config_file] + cmd = shared.get_npm_cmd('babel') + [ + filename, + '-o', + outfile, + '--presets', + '@babel/preset-env', + '--config-file', + config_file, + ] check_call(cmd, cwd=path_from_root()) return outfile @@ -564,7 +573,9 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None): # externs file for the exports, Closure is able to reason about the exports. if settings.WASM_EXPORTS and not settings.DECLARE_ASM_MODULE_EXPORTS: # Generate an exports file that records all the exported symbols from the wasm module. - module_exports_suppressions = '\n'.join(['/**\n * @suppress {duplicate, undefinedVars}\n */\nvar %s;\n' % asmjs_mangle(i) for i in settings.WASM_EXPORTS]) + module_exports_suppressions = '\n'.join( + ['/**\n * @suppress {duplicate, undefinedVars}\n */\nvar %s;\n' % asmjs_mangle(i) for i in settings.WASM_EXPORTS] + ) exports_file = shared.get_temp_files().get('.js', prefix='emcc_module_exports_') exports_file.write(module_exports_suppressions.encode()) exports_file.close() @@ -575,8 +586,7 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None): if settings.ENVIRONMENT_MAY_BE_NODE: NODE_EXTERNS_BASE = path_from_root('third_party/closure-compiler/node-externs') NODE_EXTERNS = os.listdir(NODE_EXTERNS_BASE) - NODE_EXTERNS = [os.path.join(NODE_EXTERNS_BASE, name) for name in NODE_EXTERNS - if name.endswith('.js')] + NODE_EXTERNS = [os.path.join(NODE_EXTERNS_BASE, name) for name in NODE_EXTERNS if name.endswith('.js')] CLOSURE_EXTERNS += [path_from_root('src/closure-externs/node-externs.js')] + NODE_EXTERNS # V8/SpiderMonkey shell specific externs @@ -590,8 +600,7 @@ def closure_compiler(filename, advanced=True, extra_closure_args=None): BROWSER_EXTERNS_BASE = path_from_root('src/closure-externs/browser-externs') if os.path.isdir(BROWSER_EXTERNS_BASE): BROWSER_EXTERNS = os.listdir(BROWSER_EXTERNS_BASE) - BROWSER_EXTERNS = [os.path.join(BROWSER_EXTERNS_BASE, name) for name in BROWSER_EXTERNS - if name.endswith('.js')] + BROWSER_EXTERNS = [os.path.join(BROWSER_EXTERNS_BASE, name) for name in BROWSER_EXTERNS if name.endswith('.js')] CLOSURE_EXTERNS += BROWSER_EXTERNS if settings.DYNCALLS: @@ -680,7 +689,7 @@ def move_to_safe_7bit_ascii_filename(filename): sys.stderr.write(f'{i + 1}: {input_file[i]}\n') if proc.returncode != 0: - logger.error(proc.stderr) # print list of errors (possibly long wall of text if input was minified) + logger.error(proc.stderr) # print list of errors (possibly long wall of text if input was minified) # Exit and print final hint to get clearer output msg = f'closure compiler failed (rc: {proc.returncode}): {shared.shlex_join(cmd)}' @@ -729,10 +738,7 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, debug_info): # if we are optimizing for size, shrink the combined wasm+JS # TODO: support this when a symbol map is used if expensive_optimizations: - js_file = metadce(js_file, - wasm_file, - debug_info=debug_info, - last=not settings.MINIFY_WASM_IMPORTS_AND_EXPORTS) + js_file = metadce(js_file, wasm_file, debug_info=debug_info, last=not settings.MINIFY_WASM_IMPORTS_AND_EXPORTS) # now that we removed unneeded communication between js and wasm, we can clean up # the js some more. passes = ['AJSDCE'] @@ -741,17 +747,22 @@ def minify_wasm_js(js_file, wasm_file, expensive_optimizations, debug_info): logger.debug('running post-meta-DCE cleanup on shell code: ' + ' '.join(passes)) js_file = acorn_optimizer(js_file, passes) if settings.MINIFY_WASM_IMPORTS_AND_EXPORTS: - js_file = minify_wasm_imports_and_exports(js_file, wasm_file, - minify_exports=settings.MINIFY_WASM_EXPORT_NAMES, - debug_info=debug_info) + js_file = minify_wasm_imports_and_exports( + js_file, wasm_file, minify_exports=settings.MINIFY_WASM_EXPORT_NAMES, debug_info=debug_info + ) return js_file def is_internal_global(name): - internal_start_stop_symbols = {'__start_em_asm', '__stop_em_asm', - '__start_em_js', '__stop_em_js', - '__start_em_lib_deps', '__stop_em_lib_deps', - '__em_lib_deps'} + internal_start_stop_symbols = { + '__start_em_asm', + '__stop_em_asm', + '__start_em_js', + '__stop_em_js', + '__start_em_lib_deps', + '__stop_em_lib_deps', + '__em_lib_deps', + } internal_prefixes = ('__em_js__', '__em_lib_deps') return name in internal_start_stop_symbols or any(name.startswith(p) for p in internal_prefixes) @@ -759,9 +770,7 @@ def is_internal_global(name): # get the flags to pass into the very last binaryen tool invocation, that runs # the final set of optimizations def get_last_binaryen_opts(): - return [f'--optimize-level={settings.OPT_LEVEL}', - f'--shrink-level={settings.SHRINK_LEVEL}', - '--optimize-stack-ir'] + return [f'--optimize-level={settings.OPT_LEVEL}', f'--shrink-level={settings.SHRINK_LEVEL}', '--optimize-stack-ir'] # run binaryen's wasm-metadce to dce both js and wasm @@ -838,12 +847,7 @@ def metadce(js_file, wasm_file, debug_info, last): args = ['--graph-file=' + temp] if last: args += get_last_binaryen_opts() - out = run_binaryen_command('wasm-metadce', - wasm_file, - wasm_file, - args, - debug=debug_info, - stdout=PIPE) + out = run_binaryen_command('wasm-metadce', wasm_file, wasm_file, args, debug=debug_info, stdout=PIPE) # find the unused things in js unused_imports = [] unused_exports = [] @@ -885,10 +889,7 @@ def asyncify_lazy_load_code(wasm_target, debug): args = ['--remove-memory', '--mod-asyncify-never-unwind'] if settings.OPT_LEVEL > 0: args.append(opt_level_to_str(settings.OPT_LEVEL, settings.SHRINK_LEVEL)) - run_wasm_opt(wasm_target, - wasm_target + '.lazy.wasm', - args=args, - debug=debug) + run_wasm_opt(wasm_target, wasm_target + '.lazy.wasm', args=args, debug=debug) # re-optimize the original, by applying the knowledge that imports will # definitely unwind, and we never rewind, after which optimizations can remove # a lot of code @@ -897,10 +898,7 @@ def asyncify_lazy_load_code(wasm_target, debug): args = ['--mod-asyncify-always-and-only-unwind'] if settings.OPT_LEVEL > 0: args.append(opt_level_to_str(settings.OPT_LEVEL, settings.SHRINK_LEVEL)) - run_wasm_opt(infile=wasm_target, - outfile=wasm_target, - args=args, - debug=debug) + run_wasm_opt(infile=wasm_target, outfile=wasm_target, args=args, debug=debug) def minify_wasm_imports_and_exports(js_file, wasm_file, minify_exports, debug_info): @@ -946,10 +944,7 @@ def wasm2js(js_file, wasm_file, opt_level, use_closure_compiler, debug_info, sym args += ['-O'] if symbols_file: args += ['--symbols-file=%s' % symbols_file] - wasm2js_js = run_binaryen_command('wasm2js', wasm_file, - args=args, - debug=debug_info, - stdout=PIPE) + wasm2js_js = run_binaryen_command('wasm2js', wasm_file, args=args, debug=debug_info, stdout=PIPE) if DEBUG: utils.write_file(os.path.join(get_emscripten_temp_dir(), 'wasm2js-output.js'), wasm2js_js) # JS optimizations @@ -1017,8 +1012,7 @@ def emit_debug_on_side(wasm_file, wasm_file_with_dwarf): embedded_path = settings.SEPARATE_DWARF_URL if not embedded_path: # a path was provided - make it relative to the wasm. - embedded_path = os.path.relpath(wasm_file_with_dwarf, - os.path.dirname(wasm_file)) + embedded_path = os.path.relpath(wasm_file_with_dwarf, os.path.dirname(wasm_file)) # normalize the path to use URL-style separators, per the spec embedded_path = utils.normalize_path(embedded_path) @@ -1037,12 +1031,12 @@ def emit_debug_on_side(wasm_file, wasm_file_with_dwarf): # embed a section in the main wasm to point to the file with external DWARF, # see https://yurydelendik.github.io/webassembly-dwarf/#external-DWARF - section_name = b'\x13external_debug_info' # section name, including prefixed size + section_name = b'\x13external_debug_info' # section name, including prefixed size filename_bytes = embedded_path.encode('utf-8') contents = webassembly.to_leb(len(filename_bytes)) + filename_bytes section_size = len(section_name) + len(contents) with open(wasm_file, 'ab') as f: - f.write(b'\0') # user section is code 0 + f.write(b'\0') # user section is code 0 f.write(webassembly.to_leb(section_size)) f.write(section_name) f.write(contents) @@ -1099,12 +1093,13 @@ def handle_final_wasm_symbols(wasm_file, symbols_file, debug_info): def is_ar(filename): - """Return True if a the given filename is an ar archive, False otherwise. - """ + """Return True if a the given filename is an ar archive, False otherwise.""" try: header = open(filename, 'rb').read(8) except Exception as e: - logger.debug('is_ar failed to test whether file \'%s\' is a llvm archive file! Failed on exception: %s' % (filename, e)) + logger.debug( + 'is_ar failed to test whether file \'%s\' is a llvm archive file! Failed on exception: %s' % (filename, e) + ) return False return header in (b'!\n', b'!\n') @@ -1134,11 +1129,16 @@ def emit_wasm_source_map(wasm_file, map_file, final_wasm): # source file paths must be relative to the location of the map (which is # emitted alongside the wasm) base_path = os.path.dirname(os.path.abspath(final_wasm)) - sourcemap_cmd = [sys.executable, '-E', path_from_root('tools/wasm-sourcemap.py'), - wasm_file, - '--dwarfdump=' + LLVM_DWARFDUMP, - '-o', map_file, - '--basepath=' + base_path] + sourcemap_cmd = [ + sys.executable, + '-E', + path_from_root('tools/wasm-sourcemap.py'), + wasm_file, + '--dwarfdump=' + LLVM_DWARFDUMP, + '-o', + map_file, + '--basepath=' + base_path, + ] check_call(sourcemap_cmd) @@ -1170,7 +1170,9 @@ def check_binaryen(bindir): # Allow the expected version or the following one in order avoid needing to update both # emscripten and binaryen in lock step in emscripten-releases. if version not in (EXPECTED_BINARYEN_VERSION, EXPECTED_BINARYEN_VERSION + 1): - diagnostics.warning('version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION) + diagnostics.warning( + 'version-check', 'unexpected binaryen version: %s (expected %s)', version, EXPECTED_BINARYEN_VERSION + ) def get_binaryen_bin(): @@ -1205,9 +1207,11 @@ def run_binaryen_command(tool, infile, outfile=None, args=None, debug=False, std extra += '\nnote: to disable int64 legalization (which requires changes after link) use -sWASM_BIGINT' if settings.OPT_LEVEL > 1: extra += '\nnote: -O2+ optimizations always require changes, build with -O0 or -O1 instead' - exit_with_error(f'changes to the wasm are required after link, but disallowed by ERROR_ON_WASM_CHANGES_AFTER_LINK: {cmd}{extra}') + exit_with_error( + f'changes to the wasm are required after link, but disallowed by ERROR_ON_WASM_CHANGES_AFTER_LINK: {cmd}{extra}' + ) if debug: - cmd += ['-g'] # preserve the debug info + cmd += ['-g'] # preserve the debug info # if the features are not already handled, handle them cmd += get_binaryen_feature_flags() # if we are emitting a source map, every time we load and save the wasm diff --git a/tools/cache.py b/tools/cache.py index 445de535d5683..d97c9773b88fd 100644 --- a/tools/cache.py +++ b/tools/cache.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Permanent cache for system libraries and ports. -""" +"""Permanent cache for system libraries and ports.""" import contextlib import logging @@ -35,15 +34,21 @@ def acquire_cache_lock(reason): raise Exception('Attempt to lock the cache but FROZEN_CACHE is set') if not is_writable(cachedir): - utils.exit_with_error(f'cache directory "{cachedir}" is not writable while accessing cache for: {reason} (see https://emscripten.org/docs/tools_reference/emcc.html for info on setting the cache directory)') + utils.exit_with_error( + f'cache directory "{cachedir}" is not writable while accessing cache for: {reason} (see https://emscripten.org/docs/tools_reference/emcc.html for info on setting the cache directory)' + ) if acquired_count == 0: logger.debug(f'PID {os.getpid()} acquiring multiprocess file lock to Emscripten cache at {cachedir}') - assert 'EM_CACHE_IS_LOCKED' not in os.environ, f'attempt to lock the cache while a parent process is holding the lock ({reason})' + assert ( + 'EM_CACHE_IS_LOCKED' not in os.environ + ), f'attempt to lock the cache while a parent process is holding the lock ({reason})' try: cachelock.acquire(60) except filelock.Timeout: - logger.warning(f'Accessing the Emscripten cache at "{cachedir}" (for "{reason}") is taking a long time, another process should be writing to it. If there are none and you suspect this process has deadlocked, try deleting the lock file "{cachelock_name}" and try again. If this occurs deterministically, consider filing a bug.') + logger.warning( + f'Accessing the Emscripten cache at "{cachedir}" (for "{reason}") is taking a long time, another process should be writing to it. If there are none and you suspect this process has deadlocked, try deleting the lock file "{cachelock_name}" and try again. If this occurs deterministically, consider filing a bug.' + ) cachelock.acquire() os.environ['EM_CACHE_IS_LOCKED'] = '1' @@ -78,7 +83,9 @@ def ensure(): try: utils.safe_ensure_dirs(cachedir) except Exception as e: - utils.exit_with_error(f'unable to create cache directory "{cachedir}": {e} (see https://emscripten.org/docs/tools_reference/emcc.html for info on setting the cache directory)') + utils.exit_with_error( + f'unable to create cache directory "{cachedir}": {e} (see https://emscripten.org/docs/tools_reference/emcc.html for info on setting the cache directory)' + ) def erase(): diff --git a/tools/clean_webconsole.py b/tools/clean_webconsole.py index 77b505478bf18..f57d539899cfc 100644 --- a/tools/clean_webconsole.py +++ b/tools/clean_webconsole.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Removes timestamp and line info from a webgl log -""" +"""Removes timestamp and line info from a webgl log""" import os import re @@ -23,9 +22,9 @@ def nice(x): repdata = ( - Path(path_from_root('system/include/GL/gl.h')).read_text().splitlines(keepends=True) + - ['\n'] + - Path(path_from_root('system/include/GL/glext.h')).read_text().splitlines(keepends=True) + Path(path_from_root('system/include/GL/gl.h')).read_text().splitlines(keepends=True) + + ['\n'] + + Path(path_from_root('system/include/GL/glext.h')).read_text().splitlines(keepends=True) ) reps = {} for rep in repdata: @@ -40,5 +39,9 @@ def nice(x): if line.startswith('['): line = line[15:] line = line.split(' @ ')[0] - line = re.sub(r'(0x[\dabcdef]+)', lambda hexx: reps[nice(hexx.group(0))] if nice(hexx.group(0)) in reps else nice(hexx.group(0)), line) + line = re.sub( + r'(0x[\dabcdef]+)', + lambda hexx: reps[nice(hexx.group(0))] if nice(hexx.group(0)) in reps else nice(hexx.group(0)), + line, + ) print(line) diff --git a/tools/colored_logger.py b/tools/colored_logger.py index 2802415a1ae55..50e933af4d362 100644 --- a/tools/colored_logger.py +++ b/tools/colored_logger.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Enables colored logger just by importing this module -""" +"""Enables colored logger just by importing this module""" import ctypes import sys @@ -20,16 +19,10 @@ def _get_color(): WORD = ctypes.c_ushort class COORD(ctypes.Structure): - _fields_ = [ - ("X", SHORT), - ("Y", SHORT)] + _fields_ = [("X", SHORT), ("Y", SHORT)] class SMALL_RECT(ctypes.Structure): - _fields_ = [ - ("Left", SHORT), - ("Top", SHORT), - ("Right", SHORT), - ("Bottom", SHORT)] + _fields_ = [("Left", SHORT), ("Top", SHORT), ("Right", SHORT), ("Bottom", SHORT)] class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): _fields_ = [ @@ -37,7 +30,8 @@ class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): ("dwCursorPosition", COORD), ("wAttributes", WORD), ("srWindow", SMALL_RECT), - ("dwMaximumWindowSize", COORD)] + ("dwMaximumWindowSize", COORD), + ] hdl = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) csbi = CONSOLE_SCREEN_BUFFER_INFO() @@ -50,40 +44,40 @@ def _set_color(code): def new(*args): # wincon.h - FOREGROUND_BLACK = 0x0000 # noqa - FOREGROUND_BLUE = 0x0001 # noqa - FOREGROUND_GREEN = 0x0002 # noqa - FOREGROUND_CYAN = 0x0003 # noqa - FOREGROUND_RED = 0x0004 # noqa - FOREGROUND_MAGENTA = 0x0005 # noqa - FOREGROUND_YELLOW = 0x0006 # noqa - FOREGROUND_GREY = 0x0007 # noqa - FOREGROUND_INTENSITY = 0x0008 # foreground color is intensified. - - FOREGROUND_WHITE = FOREGROUND_BLUE|FOREGROUND_GREEN |FOREGROUND_RED # noqa - - BACKGROUND_BLACK = 0x0000 # noqa - BACKGROUND_BLUE = 0x0010 # noqa - BACKGROUND_GREEN = 0x0020 # noqa - BACKGROUND_CYAN = 0x0030 # noqa - BACKGROUND_RED = 0x0040 # noqa - BACKGROUND_MAGENTA = 0x0050 # noqa - BACKGROUND_YELLOW = 0x0060 # noqa - BACKGROUND_GREY = 0x0070 # noqa - BACKGROUND_INTENSITY = 0x0080 # background color is intensified. + FOREGROUND_BLACK = 0x0000 # noqa + FOREGROUND_BLUE = 0x0001 # noqa + FOREGROUND_GREEN = 0x0002 # noqa + FOREGROUND_CYAN = 0x0003 # noqa + FOREGROUND_RED = 0x0004 # noqa + FOREGROUND_MAGENTA = 0x0005 # noqa + FOREGROUND_YELLOW = 0x0006 # noqa + FOREGROUND_GREY = 0x0007 # noqa + FOREGROUND_INTENSITY = 0x0008 # foreground color is intensified. + + FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED # noqa + + BACKGROUND_BLACK = 0x0000 # noqa + BACKGROUND_BLUE = 0x0010 # noqa + BACKGROUND_GREEN = 0x0020 # noqa + BACKGROUND_CYAN = 0x0030 # noqa + BACKGROUND_RED = 0x0040 # noqa + BACKGROUND_MAGENTA = 0x0050 # noqa + BACKGROUND_YELLOW = 0x0060 # noqa + BACKGROUND_GREY = 0x0070 # noqa + BACKGROUND_INTENSITY = 0x0080 # background color is intensified. levelno = args[1].levelno - if (levelno >= 50): - color = BACKGROUND_YELLOW | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_INTENSITY - elif (levelno >= 40): - color = FOREGROUND_RED | FOREGROUND_INTENSITY - elif (levelno >= 30): - color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY - elif (levelno >= 20): - color = FOREGROUND_GREEN - elif (levelno >= 10): - color = FOREGROUND_MAGENTA + if levelno >= 50: + color = BACKGROUND_YELLOW | FOREGROUND_RED | FOREGROUND_INTENSITY | BACKGROUND_INTENSITY + elif levelno >= 40: + color = FOREGROUND_RED | FOREGROUND_INTENSITY + elif levelno >= 30: + color = FOREGROUND_YELLOW | FOREGROUND_INTENSITY + elif levelno >= 20: + color = FOREGROUND_GREEN + elif levelno >= 10: + color = FOREGROUND_MAGENTA else: - color = FOREGROUND_WHITE + color = FOREGROUND_WHITE old_color = _get_color() _set_color(color) @@ -100,15 +94,15 @@ def add_coloring_to_emit_ansi(fn): def new(*args): levelno = args[1].levelno if levelno >= 40: - color = '\x1b[31m' # red + color = '\x1b[31m' # red elif levelno >= 30: - color = '\x1b[33m' # yellow + color = '\x1b[33m' # yellow elif levelno >= 20: - color = '\x1b[32m' # green + color = '\x1b[32m' # green elif levelno >= 10: - color = '\x1b[35m' # pink + color = '\x1b[35m' # pink else: - color = '\x1b[0m' # normal + color = '\x1b[0m' # normal args[1].msg = color + args[1].msg + '\x1b[0m' # normal return fn(*args) diff --git a/tools/config.py b/tools/config.py index 43c738cf8feb0..aa1751f64fb5b 100644 --- a/tools/config.py +++ b/tools/config.py @@ -87,7 +87,9 @@ def set_config_from_tool_location(config_key, tool_binary, f): path = shutil.which(tool_binary) if not path: if not os.path.isfile(EM_CONFIG): - diagnostics.warn('config file not found: %s. You can create one by hand or run `emcc --generate-config`', EM_CONFIG) + diagnostics.warn( + 'config file not found: %s. You can create one by hand or run `emcc --generate-config`', EM_CONFIG + ) exit_with_error('%s not set in config (%s), and `%s` not found in PATH', config_key, EM_CONFIG, tool_binary) globals()[config_key] = f(path) elif not val: @@ -182,7 +184,7 @@ def generate_config(path): # The full string is replaced so that the template stays valid Python. config_data = utils.read_file(path_from_root('tools/config_template.py')) - config_data = config_data.splitlines()[3:] # remove the initial comment + config_data = config_data.splitlines()[3:] # remove the initial comment config_data = '\n'.join(config_data) + '\n' # autodetect some default paths llvm_root = os.path.dirname(shutil.which('wasm-ld') or '/usr/bin/wasm-ld') @@ -197,7 +199,8 @@ def generate_config(path): # write utils.write_file(path, config_data) - print('''\ + print( + '''\ An Emscripten settings file has been generated at: %s @@ -209,7 +212,10 @@ def generate_config(path): NODE_JS = %s Please edit the file if any of those are incorrect.\ -''' % (path, llvm_root, binaryen_root, node), file=sys.stderr) +''' + % (path, llvm_root, binaryen_root, node), + file=sys.stderr, + ) def find_config_file(): diff --git a/tools/config_template.py b/tools/config_template.py index 195593ccefa7f..8d415a31f750f 100644 --- a/tools/config_template.py +++ b/tools/config_template.py @@ -12,12 +12,12 @@ # is not valid, but LLVM='c:\\llvm\\' and LLVM='c:/llvm/' # are. -LLVM_ROOT = '{{{ LLVM_ROOT }}}' # directory -BINARYEN_ROOT = '{{{ BINARYEN_ROOT }}}' # directory +LLVM_ROOT = '{{{ LLVM_ROOT }}}' # directory +BINARYEN_ROOT = '{{{ BINARYEN_ROOT }}}' # directory # Location of the node binary to use for running the JS parts of the compiler. # This engine must exist, or nothing can be compiled. -NODE_JS = '{{{ NODE }}}' # executable +NODE_JS = '{{{ NODE }}}' # executable ################################################################################ # diff --git a/tools/create_dom_pk_codes.py b/tools/create_dom_pk_codes.py index c6fc0a0b2be03..d6a6cee79d572 100755 --- a/tools/create_dom_pk_codes.py +++ b/tools/create_dom_pk_codes.py @@ -34,96 +34,96 @@ import random input_strings = [ - (0x0, 'Unidentified', 'DOM_PK_UNKNOWN'), - (0x1, 'Escape', 'DOM_PK_ESCAPE'), - (0x2, 'Digit0', 'DOM_PK_0'), - (0x3, 'Digit1', 'DOM_PK_1'), - (0x4, 'Digit2', 'DOM_PK_2'), - (0x5, 'Digit3', 'DOM_PK_3'), - (0x6, 'Digit4', 'DOM_PK_4'), - (0x7, 'Digit5', 'DOM_PK_5'), - (0x8, 'Digit6', 'DOM_PK_6'), - (0x9, 'Digit7', 'DOM_PK_7'), - (0xA, 'Digit8', 'DOM_PK_8'), - (0xB, 'Digit9', 'DOM_PK_9'), - (0xC, 'Minus', 'DOM_PK_MINUS'), - (0xD, 'Equal', 'DOM_PK_EQUAL'), - (0xE, 'Backspace', 'DOM_PK_BACKSPACE'), - (0xF, 'Tab', 'DOM_PK_TAB'), - (0x10, 'KeyQ', 'DOM_PK_Q'), - (0x11, 'KeyW', 'DOM_PK_W'), - (0x12, 'KeyE', 'DOM_PK_E'), - (0x13, 'KeyR', 'DOM_PK_R'), - (0x14, 'KeyT', 'DOM_PK_T'), - (0x15, 'KeyY', 'DOM_PK_Y'), - (0x16, 'KeyU', 'DOM_PK_U'), - (0x17, 'KeyI', 'DOM_PK_I'), - (0x18, 'KeyO', 'DOM_PK_O'), - (0x19, 'KeyP', 'DOM_PK_P'), - (0x1A, 'BracketLeft', 'DOM_PK_BRACKET_LEFT'), - (0x1B, 'BracketRight', 'DOM_PK_BRACKET_RIGHT'), - (0x1C, 'Enter', 'DOM_PK_ENTER'), - (0x1D, 'ControlLeft', 'DOM_PK_CONTROL_LEFT'), - (0x1E, 'KeyA', 'DOM_PK_A'), - (0x1F, 'KeyS', 'DOM_PK_S'), - (0x20, 'KeyD', 'DOM_PK_D'), - (0x21, 'KeyF', 'DOM_PK_F'), - (0x22, 'KeyG', 'DOM_PK_G'), - (0x23, 'KeyH', 'DOM_PK_H'), - (0x24, 'KeyJ', 'DOM_PK_J'), - (0x25, 'KeyK', 'DOM_PK_K'), - (0x26, 'KeyL', 'DOM_PK_L'), - (0x27, 'Semicolon', 'DOM_PK_SEMICOLON'), - (0x28, 'Quote', 'DOM_PK_QUOTE'), - (0x29, 'Backquote', 'DOM_PK_BACKQUOTE'), - (0x2A, 'ShiftLeft', 'DOM_PK_SHIFT_LEFT'), - (0x2B, 'Backslash', 'DOM_PK_BACKSLASH'), - (0x2C, 'KeyZ', 'DOM_PK_Z'), - (0x2D, 'KeyX', 'DOM_PK_X'), - (0x2E, 'KeyC', 'DOM_PK_C'), - (0x2F, 'KeyV', 'DOM_PK_V'), - (0x30, 'KeyB', 'DOM_PK_B'), - (0x31, 'KeyN', 'DOM_PK_N'), - (0x32, 'KeyM', 'DOM_PK_M'), - (0x33, 'Comma', 'DOM_PK_COMMA'), - (0x34, 'Period', 'DOM_PK_PERIOD'), - (0x35, 'Slash', 'DOM_PK_SLASH'), - (0x36, 'ShiftRight', 'DOM_PK_SHIFT_RIGHT'), - (0x37, 'NumpadMultiply', 'DOM_PK_NUMPAD_MULTIPLY'), - (0x38, 'AltLeft', 'DOM_PK_ALT_LEFT'), - (0x39, 'Space', 'DOM_PK_SPACE'), - (0x3A, 'CapsLock', 'DOM_PK_CAPS_LOCK'), - (0x3B, 'F1', 'DOM_PK_F1'), - (0x3C, 'F2', 'DOM_PK_F2'), - (0x3D, 'F3', 'DOM_PK_F3'), - (0x3E, 'F4', 'DOM_PK_F4'), - (0x3F, 'F5', 'DOM_PK_F5'), - (0x40, 'F6', 'DOM_PK_F6'), - (0x41, 'F7', 'DOM_PK_F7'), - (0x42, 'F8', 'DOM_PK_F8'), - (0x43, 'F9', 'DOM_PK_F9'), - (0x44, 'F10', 'DOM_PK_F10'), - (0x45, 'Pause', 'DOM_PK_PAUSE'), - (0x46, 'ScrollLock', 'DOM_PK_SCROLL_LOCK'), - (0x47, 'Numpad7', 'DOM_PK_NUMPAD_7'), - (0x48, 'Numpad8', 'DOM_PK_NUMPAD_8'), - (0x49, 'Numpad9', 'DOM_PK_NUMPAD_9'), - (0x4A, 'NumpadSubtract', 'DOM_PK_NUMPAD_SUBTRACT'), - (0x4B, 'Numpad4', 'DOM_PK_NUMPAD_4'), - (0x4C, 'Numpad5', 'DOM_PK_NUMPAD_5'), - (0x4D, 'Numpad6', 'DOM_PK_NUMPAD_6'), - (0x4E, 'NumpadAdd', 'DOM_PK_NUMPAD_ADD'), - (0x4F, 'Numpad1', 'DOM_PK_NUMPAD_1'), - (0x50, 'Numpad2', 'DOM_PK_NUMPAD_2'), - (0x51, 'Numpad3', 'DOM_PK_NUMPAD_3'), - (0x52, 'Numpad0', 'DOM_PK_NUMPAD_0'), - (0x53, 'NumpadDecimal', 'DOM_PK_NUMPAD_DECIMAL'), - (0x54, 'PrintScreen', 'DOM_PK_PRINT_SCREEN'), + (0x0, 'Unidentified', 'DOM_PK_UNKNOWN'), + (0x1, 'Escape', 'DOM_PK_ESCAPE'), + (0x2, 'Digit0', 'DOM_PK_0'), + (0x3, 'Digit1', 'DOM_PK_1'), + (0x4, 'Digit2', 'DOM_PK_2'), + (0x5, 'Digit3', 'DOM_PK_3'), + (0x6, 'Digit4', 'DOM_PK_4'), + (0x7, 'Digit5', 'DOM_PK_5'), + (0x8, 'Digit6', 'DOM_PK_6'), + (0x9, 'Digit7', 'DOM_PK_7'), + (0xA, 'Digit8', 'DOM_PK_8'), + (0xB, 'Digit9', 'DOM_PK_9'), + (0xC, 'Minus', 'DOM_PK_MINUS'), + (0xD, 'Equal', 'DOM_PK_EQUAL'), + (0xE, 'Backspace', 'DOM_PK_BACKSPACE'), + (0xF, 'Tab', 'DOM_PK_TAB'), + (0x10, 'KeyQ', 'DOM_PK_Q'), + (0x11, 'KeyW', 'DOM_PK_W'), + (0x12, 'KeyE', 'DOM_PK_E'), + (0x13, 'KeyR', 'DOM_PK_R'), + (0x14, 'KeyT', 'DOM_PK_T'), + (0x15, 'KeyY', 'DOM_PK_Y'), + (0x16, 'KeyU', 'DOM_PK_U'), + (0x17, 'KeyI', 'DOM_PK_I'), + (0x18, 'KeyO', 'DOM_PK_O'), + (0x19, 'KeyP', 'DOM_PK_P'), + (0x1A, 'BracketLeft', 'DOM_PK_BRACKET_LEFT'), + (0x1B, 'BracketRight', 'DOM_PK_BRACKET_RIGHT'), + (0x1C, 'Enter', 'DOM_PK_ENTER'), + (0x1D, 'ControlLeft', 'DOM_PK_CONTROL_LEFT'), + (0x1E, 'KeyA', 'DOM_PK_A'), + (0x1F, 'KeyS', 'DOM_PK_S'), + (0x20, 'KeyD', 'DOM_PK_D'), + (0x21, 'KeyF', 'DOM_PK_F'), + (0x22, 'KeyG', 'DOM_PK_G'), + (0x23, 'KeyH', 'DOM_PK_H'), + (0x24, 'KeyJ', 'DOM_PK_J'), + (0x25, 'KeyK', 'DOM_PK_K'), + (0x26, 'KeyL', 'DOM_PK_L'), + (0x27, 'Semicolon', 'DOM_PK_SEMICOLON'), + (0x28, 'Quote', 'DOM_PK_QUOTE'), + (0x29, 'Backquote', 'DOM_PK_BACKQUOTE'), + (0x2A, 'ShiftLeft', 'DOM_PK_SHIFT_LEFT'), + (0x2B, 'Backslash', 'DOM_PK_BACKSLASH'), + (0x2C, 'KeyZ', 'DOM_PK_Z'), + (0x2D, 'KeyX', 'DOM_PK_X'), + (0x2E, 'KeyC', 'DOM_PK_C'), + (0x2F, 'KeyV', 'DOM_PK_V'), + (0x30, 'KeyB', 'DOM_PK_B'), + (0x31, 'KeyN', 'DOM_PK_N'), + (0x32, 'KeyM', 'DOM_PK_M'), + (0x33, 'Comma', 'DOM_PK_COMMA'), + (0x34, 'Period', 'DOM_PK_PERIOD'), + (0x35, 'Slash', 'DOM_PK_SLASH'), + (0x36, 'ShiftRight', 'DOM_PK_SHIFT_RIGHT'), + (0x37, 'NumpadMultiply', 'DOM_PK_NUMPAD_MULTIPLY'), + (0x38, 'AltLeft', 'DOM_PK_ALT_LEFT'), + (0x39, 'Space', 'DOM_PK_SPACE'), + (0x3A, 'CapsLock', 'DOM_PK_CAPS_LOCK'), + (0x3B, 'F1', 'DOM_PK_F1'), + (0x3C, 'F2', 'DOM_PK_F2'), + (0x3D, 'F3', 'DOM_PK_F3'), + (0x3E, 'F4', 'DOM_PK_F4'), + (0x3F, 'F5', 'DOM_PK_F5'), + (0x40, 'F6', 'DOM_PK_F6'), + (0x41, 'F7', 'DOM_PK_F7'), + (0x42, 'F8', 'DOM_PK_F8'), + (0x43, 'F9', 'DOM_PK_F9'), + (0x44, 'F10', 'DOM_PK_F10'), + (0x45, 'Pause', 'DOM_PK_PAUSE'), + (0x46, 'ScrollLock', 'DOM_PK_SCROLL_LOCK'), + (0x47, 'Numpad7', 'DOM_PK_NUMPAD_7'), + (0x48, 'Numpad8', 'DOM_PK_NUMPAD_8'), + (0x49, 'Numpad9', 'DOM_PK_NUMPAD_9'), + (0x4A, 'NumpadSubtract', 'DOM_PK_NUMPAD_SUBTRACT'), + (0x4B, 'Numpad4', 'DOM_PK_NUMPAD_4'), + (0x4C, 'Numpad5', 'DOM_PK_NUMPAD_5'), + (0x4D, 'Numpad6', 'DOM_PK_NUMPAD_6'), + (0x4E, 'NumpadAdd', 'DOM_PK_NUMPAD_ADD'), + (0x4F, 'Numpad1', 'DOM_PK_NUMPAD_1'), + (0x50, 'Numpad2', 'DOM_PK_NUMPAD_2'), + (0x51, 'Numpad3', 'DOM_PK_NUMPAD_3'), + (0x52, 'Numpad0', 'DOM_PK_NUMPAD_0'), + (0x53, 'NumpadDecimal', 'DOM_PK_NUMPAD_DECIMAL'), + (0x54, 'PrintScreen', 'DOM_PK_PRINT_SCREEN'), # 0x0055 'Unidentified', '' - (0x56, 'IntlBackslash', 'DOM_PK_INTL_BACKSLASH'), - (0x57, 'F11', 'DOM_PK_F11'), - (0x58, 'F12', 'DOM_PK_F12'), - (0x59, 'NumpadEqual', 'DOM_PK_NUMPAD_EQUAL'), + (0x56, 'IntlBackslash', 'DOM_PK_INTL_BACKSLASH'), + (0x57, 'F11', 'DOM_PK_F11'), + (0x58, 'F12', 'DOM_PK_F12'), + (0x59, 'NumpadEqual', 'DOM_PK_NUMPAD_EQUAL'), # 0x005A 'Unidentified', '' # 0x005B 'Unidentified', '' # 0x005C 'Unidentified', '' @@ -134,84 +134,84 @@ # 0x0061 'Unidentified', '' # 0x0062 'Unidentified', '' # 0x0063 'Unidentified', '' - (0x64, 'F13', 'DOM_PK_F13'), - (0x65, 'F14', 'DOM_PK_F14'), - (0x66, 'F15', 'DOM_PK_F15'), - (0x67, 'F16', 'DOM_PK_F16'), - (0x68, 'F17', 'DOM_PK_F17'), - (0x69, 'F18', 'DOM_PK_F18'), - (0x6A, 'F19', 'DOM_PK_F19'), - (0x6B, 'F20', 'DOM_PK_F20'), - (0x6C, 'F21', 'DOM_PK_F21'), - (0x6D, 'F22', 'DOM_PK_F22'), - (0x6E, 'F23', 'DOM_PK_F23'), + (0x64, 'F13', 'DOM_PK_F13'), + (0x65, 'F14', 'DOM_PK_F14'), + (0x66, 'F15', 'DOM_PK_F15'), + (0x67, 'F16', 'DOM_PK_F16'), + (0x68, 'F17', 'DOM_PK_F17'), + (0x69, 'F18', 'DOM_PK_F18'), + (0x6A, 'F19', 'DOM_PK_F19'), + (0x6B, 'F20', 'DOM_PK_F20'), + (0x6C, 'F21', 'DOM_PK_F21'), + (0x6D, 'F22', 'DOM_PK_F22'), + (0x6E, 'F23', 'DOM_PK_F23'), # 0x006F 'Unidentified', '' - (0x70, 'KanaMode', 'DOM_PK_KANA_MODE'), - (0x71, 'Lang2', 'DOM_PK_LANG_2'), - (0x72, 'Lang1', 'DOM_PK_LANG_1'), - (0x73, 'IntlRo', 'DOM_PK_INTL_RO'), + (0x70, 'KanaMode', 'DOM_PK_KANA_MODE'), + (0x71, 'Lang2', 'DOM_PK_LANG_2'), + (0x72, 'Lang1', 'DOM_PK_LANG_1'), + (0x73, 'IntlRo', 'DOM_PK_INTL_RO'), # 0x0074 'Unidentified', '' # 0x0075 'Unidentified', '' - (0x76, 'F24', 'DOM_PK_F24'), + (0x76, 'F24', 'DOM_PK_F24'), # 0x0077 'Unidentified', '' # 0x0078 'Unidentified', '' - (0x79, 'Convert', 'DOM_PK_CONVERT'), + (0x79, 'Convert', 'DOM_PK_CONVERT'), # 0x007A 'Unidentified', '' - (0x7B, 'NonConvert', 'DOM_PK_NON_CONVERT'), + (0x7B, 'NonConvert', 'DOM_PK_NON_CONVERT'), # 0x007C 'Unidentified', '' - (0x7D, 'IntlYen', 'DOM_PK_INTL_YEN'), - (0x7E, 'NumpadComma', 'DOM_PK_NUMPAD_COMMA'), + (0x7D, 'IntlYen', 'DOM_PK_INTL_YEN'), + (0x7E, 'NumpadComma', 'DOM_PK_NUMPAD_COMMA'), # 0x007F 'Unidentified', '' - (0xE00A, 'Paste', 'DOM_PK_PASTE'), + (0xE00A, 'Paste', 'DOM_PK_PASTE'), (0xE010, 'MediaTrackPrevious', 'DOM_PK_MEDIA_TRACK_PREVIOUS'), - (0xE017, 'Cut', 'DOM_PK_CUT'), - (0xE018, 'Copy', 'DOM_PK_COPY'), - (0xE019, 'MediaTrackNext', 'DOM_PK_MEDIA_TRACK_NEXT'), - (0xE01C, 'NumpadEnter', 'DOM_PK_NUMPAD_ENTER'), - (0xE01D, 'ControlRight', 'DOM_PK_CONTROL_RIGHT'), - (0xE020, 'AudioVolumeMute', 'DOM_PK_AUDIO_VOLUME_MUTE'), - (0xE020, 'VolumeMute', 'DOM_PK_AUDIO_VOLUME_MUTE', 'duplicate'), - (0xE021, 'LaunchApp2', 'DOM_PK_LAUNCH_APP_2'), - (0xE022, 'MediaPlayPause', 'DOM_PK_MEDIA_PLAY_PAUSE'), - (0xE024, 'MediaStop', 'DOM_PK_MEDIA_STOP'), - (0xE02C, 'Eject', 'DOM_PK_EJECT'), - (0xE02E, 'AudioVolumeDown', 'DOM_PK_AUDIO_VOLUME_DOWN'), - (0xE02E, 'VolumeDown', 'DOM_PK_AUDIO_VOLUME_DOWN', 'duplicate'), - (0xE030, 'AudioVolumeUp', 'DOM_PK_AUDIO_VOLUME_UP'), - (0xE030, 'VolumeUp', 'DOM_PK_AUDIO_VOLUME_UP', 'duplicate'), - (0xE032, 'BrowserHome', 'DOM_PK_BROWSER_HOME'), - (0xE035, 'NumpadDivide', 'DOM_PK_NUMPAD_DIVIDE'), + (0xE017, 'Cut', 'DOM_PK_CUT'), + (0xE018, 'Copy', 'DOM_PK_COPY'), + (0xE019, 'MediaTrackNext', 'DOM_PK_MEDIA_TRACK_NEXT'), + (0xE01C, 'NumpadEnter', 'DOM_PK_NUMPAD_ENTER'), + (0xE01D, 'ControlRight', 'DOM_PK_CONTROL_RIGHT'), + (0xE020, 'AudioVolumeMute', 'DOM_PK_AUDIO_VOLUME_MUTE'), + (0xE020, 'VolumeMute', 'DOM_PK_AUDIO_VOLUME_MUTE', 'duplicate'), + (0xE021, 'LaunchApp2', 'DOM_PK_LAUNCH_APP_2'), + (0xE022, 'MediaPlayPause', 'DOM_PK_MEDIA_PLAY_PAUSE'), + (0xE024, 'MediaStop', 'DOM_PK_MEDIA_STOP'), + (0xE02C, 'Eject', 'DOM_PK_EJECT'), + (0xE02E, 'AudioVolumeDown', 'DOM_PK_AUDIO_VOLUME_DOWN'), + (0xE02E, 'VolumeDown', 'DOM_PK_AUDIO_VOLUME_DOWN', 'duplicate'), + (0xE030, 'AudioVolumeUp', 'DOM_PK_AUDIO_VOLUME_UP'), + (0xE030, 'VolumeUp', 'DOM_PK_AUDIO_VOLUME_UP', 'duplicate'), + (0xE032, 'BrowserHome', 'DOM_PK_BROWSER_HOME'), + (0xE035, 'NumpadDivide', 'DOM_PK_NUMPAD_DIVIDE'), # (0xE037, 'PrintScreen', 'DOM_PK_PRINT_SCREEN'), - (0xE038, 'AltRight', 'DOM_PK_ALT_RIGHT'), - (0xE03B, 'Help', 'DOM_PK_HELP'), - (0xE045, 'NumLock', 'DOM_PK_NUM_LOCK'), + (0xE038, 'AltRight', 'DOM_PK_ALT_RIGHT'), + (0xE03B, 'Help', 'DOM_PK_HELP'), + (0xE045, 'NumLock', 'DOM_PK_NUM_LOCK'), # (0xE046, 'Pause', 'DOM_PK_'), # Says Ctrl+Pause - (0xE047, 'Home', 'DOM_PK_HOME'), - (0xE048, 'ArrowUp', 'DOM_PK_ARROW_UP'), - (0xE049, 'PageUp', 'DOM_PK_PAGE_UP'), - (0xE04B, 'ArrowLeft', 'DOM_PK_ARROW_LEFT'), - (0xE04D, 'ArrowRight', 'DOM_PK_ARROW_RIGHT'), - (0xE04F, 'End', 'DOM_PK_END'), - (0xE050, 'ArrowDown', 'DOM_PK_ARROW_DOWN'), - (0xE051, 'PageDown', 'DOM_PK_PAGE_DOWN'), - (0xE052, 'Insert', 'DOM_PK_INSERT'), - (0xE053, 'Delete', 'DOM_PK_DELETE'), - (0xE05B, 'MetaLeft', 'DOM_PK_META_LEFT'), - (0xE05B, 'OSLeft', 'DOM_PK_OS_LEFT', 'duplicate'), - (0xE05C, 'MetaRight', 'DOM_PK_META_RIGHT'), - (0xE05C, 'OSRight', 'DOM_PK_OS_RIGHT', 'duplicate'), - (0xE05D, 'ContextMenu', 'DOM_PK_CONTEXT_MENU'), - (0xE05E, 'Power', 'DOM_PK_POWER'), - (0xE065, 'BrowserSearch', 'DOM_PK_BROWSER_SEARCH'), - (0xE066, 'BrowserFavorites', 'DOM_PK_BROWSER_FAVORITES'), - (0xE067, 'BrowserRefresh', 'DOM_PK_BROWSER_REFRESH'), - (0xE068, 'BrowserStop', 'DOM_PK_BROWSER_STOP'), - (0xE069, 'BrowserForward', 'DOM_PK_BROWSER_FORWARD'), - (0xE06A, 'BrowserBack', 'DOM_PK_BROWSER_BACK'), - (0xE06B, 'LaunchApp1', 'DOM_PK_LAUNCH_APP_1'), - (0xE06C, 'LaunchMail', 'DOM_PK_LAUNCH_MAIL'), - (0xE06D, 'LaunchMediaPlayer', 'DOM_PK_LAUNCH_MEDIA_PLAYER'), - (0xE06D, 'MediaSelect', 'DOM_PK_MEDIA_SELECT', 'duplicate') + (0xE047, 'Home', 'DOM_PK_HOME'), + (0xE048, 'ArrowUp', 'DOM_PK_ARROW_UP'), + (0xE049, 'PageUp', 'DOM_PK_PAGE_UP'), + (0xE04B, 'ArrowLeft', 'DOM_PK_ARROW_LEFT'), + (0xE04D, 'ArrowRight', 'DOM_PK_ARROW_RIGHT'), + (0xE04F, 'End', 'DOM_PK_END'), + (0xE050, 'ArrowDown', 'DOM_PK_ARROW_DOWN'), + (0xE051, 'PageDown', 'DOM_PK_PAGE_DOWN'), + (0xE052, 'Insert', 'DOM_PK_INSERT'), + (0xE053, 'Delete', 'DOM_PK_DELETE'), + (0xE05B, 'MetaLeft', 'DOM_PK_META_LEFT'), + (0xE05B, 'OSLeft', 'DOM_PK_OS_LEFT', 'duplicate'), + (0xE05C, 'MetaRight', 'DOM_PK_META_RIGHT'), + (0xE05C, 'OSRight', 'DOM_PK_OS_RIGHT', 'duplicate'), + (0xE05D, 'ContextMenu', 'DOM_PK_CONTEXT_MENU'), + (0xE05E, 'Power', 'DOM_PK_POWER'), + (0xE065, 'BrowserSearch', 'DOM_PK_BROWSER_SEARCH'), + (0xE066, 'BrowserFavorites', 'DOM_PK_BROWSER_FAVORITES'), + (0xE067, 'BrowserRefresh', 'DOM_PK_BROWSER_REFRESH'), + (0xE068, 'BrowserStop', 'DOM_PK_BROWSER_STOP'), + (0xE069, 'BrowserForward', 'DOM_PK_BROWSER_FORWARD'), + (0xE06A, 'BrowserBack', 'DOM_PK_BROWSER_BACK'), + (0xE06B, 'LaunchApp1', 'DOM_PK_LAUNCH_APP_1'), + (0xE06C, 'LaunchMail', 'DOM_PK_LAUNCH_MAIL'), + (0xE06D, 'LaunchMediaPlayer', 'DOM_PK_LAUNCH_MEDIA_PLAYER'), + (0xE06D, 'MediaSelect', 'DOM_PK_MEDIA_SELECT', 'duplicate'), # (0xE0F1, 'Lang2', 'DOM_PK_'), Hanja key # (0xE0F2, 'Lang2', 'DOM_PK_'), Han/Yeong ] @@ -315,7 +315,12 @@ def longest_key_code_length(): ''') for s in input_strings: - h_file.write('#define ' + pad_to_length(s[2], longest_dom_pk_code_length()) + ' 0x%04X /* "%s */' % (s[0], pad_to_length(s[1] + '"', longest_key_code_length() + 1)) + '\n') + h_file.write( + '#define ' + + pad_to_length(s[2], longest_dom_pk_code_length()) + + ' 0x%04X /* "%s */' % (s[0], pad_to_length(s[1] + '"', longest_key_code_length() + 1)) + + '\n' + ) h_file.write(''' #ifdef __cplusplus @@ -331,7 +336,8 @@ def longest_key_code_length(): #endif ''') -c_file.write(''' +c_file.write( + ''' DOM_PK_CODE_TYPE emscripten_compute_dom_pk_code(const char *keyCodeString) { if (!keyCodeString) return 0; @@ -346,10 +352,21 @@ def longest_key_code_length(): https://developer.mozilla.org/en-US/docs/Web/API/KeyboardEvent/code */ switch(hash) { -''' % (k1, k2)) +''' + % (k1, k2) +) for s in input_strings: - c_file.write(' case 0x%08XU /* %s */: return %s /* 0x%04X */' % (str_to_hash[s[1]], pad_to_length(s[1], longest_key_code_length()), pad_to_length(s[2] + ';', longest_dom_pk_code_length() + 1), s[0]) + '\n') + c_file.write( + ' case 0x%08XU /* %s */: return %s /* 0x%04X */' + % ( + str_to_hash[s[1]], + pad_to_length(s[1], longest_key_code_length()), + pad_to_length(s[2] + ';', longest_dom_pk_code_length() + 1), + s[0], + ) + + '\n' + ) c_file.write(''' default: return DOM_PK_UNKNOWN; } @@ -363,7 +380,9 @@ def longest_key_code_length(): for s in input_strings: if len(s) == 3: - c_file.write(' case %s return "%s";' % (pad_to_length(s[2] + ':', longest_dom_pk_code_length() + 1), s[2]) + '\n') + c_file.write( + ' case %s return "%s";' % (pad_to_length(s[2] + ':', longest_dom_pk_code_length() + 1), s[2]) + '\n' + ) c_file.write(''' default: return "Unknown DOM_PK code"; } diff --git a/tools/determinism_checker.py b/tools/determinism_checker.py index 03c63a52640c3..b8fa758c802c1 100644 --- a/tools/determinism_checker.py +++ b/tools/determinism_checker.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Runs a build command many times to search for any nondeterminism. -""" +"""Runs a build command many times to search for any nondeterminism.""" import os import random @@ -49,7 +48,7 @@ def write(data, subdir): while 1: print(i) i += 1 - time.sleep(random.random() / (10 * 5)) # add some timing nondeterminism here, not that we need it, but whatever + time.sleep(random.random() / (10 * 5)) # add some timing nondeterminism here, not that we need it, but whatever curr = run() if first != curr: print('NONDETERMINISM!!!1') diff --git a/tools/diagnostics.py b/tools/diagnostics.py index 608d12e40eb42..c9cb17e8ad4bb 100644 --- a/tools/diagnostics.py +++ b/tools/diagnostics.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Simple color-enabled diagnositics reporting functions. -""" +"""Simple color-enabled diagnositics reporting functions.""" import ctypes import logging @@ -35,13 +34,13 @@ # color for use for each diagnostic level level_colors = { - WARN: MAGENTA, - ERROR: RED, + WARN: MAGENTA, + ERROR: RED, } level_prefixes = { - WARN: 'warning: ', - ERROR: 'error: ', + WARN: 'warning: ', + ERROR: 'error: ', } # Constants from the Windows API @@ -51,14 +50,14 @@ def output_color_windows(color): # TODO(sbc): This code is duplicated in colored_logger.py. Refactor. # wincon.h - FOREGROUND_BLACK = 0x0000 # noqa - FOREGROUND_BLUE = 0x0001 # noqa - FOREGROUND_GREEN = 0x0002 # noqa - FOREGROUND_CYAN = 0x0003 # noqa - FOREGROUND_RED = 0x0004 # noqa - FOREGROUND_MAGENTA = 0x0005 # noqa - FOREGROUND_YELLOW = 0x0006 # noqa - FOREGROUND_GREY = 0x0007 # noqa + FOREGROUND_BLACK = 0x0000 # noqa + FOREGROUND_BLUE = 0x0001 # noqa + FOREGROUND_GREEN = 0x0002 # noqa + FOREGROUND_CYAN = 0x0003 # noqa + FOREGROUND_RED = 0x0004 # noqa + FOREGROUND_MAGENTA = 0x0005 # noqa + FOREGROUND_YELLOW = 0x0006 # noqa + FOREGROUND_GREY = 0x0007 # noqa color_map = { RED: FOREGROUND_RED, @@ -67,7 +66,7 @@ def output_color_windows(color): BLUE: FOREGROUND_BLUE, MAGENTA: FOREGROUND_MAGENTA, CYAN: FOREGROUND_CYAN, - WHITE: FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED + WHITE: FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED, } sys.stderr.flush() @@ -80,16 +79,10 @@ def get_color_windows(): WORD = ctypes.c_ushort class COORD(ctypes.Structure): - _fields_ = [ - ("X", SHORT), - ("Y", SHORT)] + _fields_ = [("X", SHORT), ("Y", SHORT)] class SMALL_RECT(ctypes.Structure): - _fields_ = [ - ("Left", SHORT), - ("Top", SHORT), - ("Right", SHORT), - ("Bottom", SHORT)] + _fields_ = [("Left", SHORT), ("Top", SHORT), ("Right", SHORT), ("Bottom", SHORT)] class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): _fields_ = [ @@ -97,7 +90,8 @@ class CONSOLE_SCREEN_BUFFER_INFO(ctypes.Structure): ("dwCursorPosition", COORD), ("wAttributes", WORD), ("srWindow", SMALL_RECT), - ("dwMaximumWindowSize", COORD)] + ("dwMaximumWindowSize", COORD), + ] hdl = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) csbi = CONSOLE_SCREEN_BUFFER_INFO() diff --git a/tools/emcoverage.py b/tools/emcoverage.py index 55d9d4aebd173..879ad5e7ac0f6 100755 --- a/tools/emcoverage.py +++ b/tools/emcoverage.py @@ -36,7 +36,7 @@ import uuid from glob import glob -import coverage.cmdline # type: ignore +import coverage.cmdline # type: ignore import contextlib SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/tools/emdump.py b/tools/emdump.py index 089d2a288df4a..00de0afdfb417 100755 --- a/tools/emdump.py +++ b/tools/emdump.py @@ -6,8 +6,7 @@ # -*- Mode: python -*- -"""emdump.py prints out statistics about compiled code sizes -""" +"""emdump.py prints out statistics about compiled code sizes""" import argparse import os @@ -58,9 +57,9 @@ def idx_to_line_col(s, i): def parse_parens(s): brace_map = {} - parens = [] # () - brackets = [] # [] - braces = [] # {} + parens = [] # () + brackets = [] # [] + braces = [] # {} i = 0 end = len(s) @@ -83,7 +82,9 @@ def parse_parens(s): # prev = i i = find_unescaped_end(s, "'", i, end) # print(idx_to_line_col(s, prev) + ' is a \'\' string, skipping to ' + idx_to_line_col(s, i)) - elif ch == '^': # Ignore parens/brackets/braces if the previous character was a '^'. This is a bit of a heuristic, '^)' occur commonly in Emscripten generated regexes + elif ( + ch == '^' + ): # Ignore parens/brackets/braces if the previous character was a '^'. This is a bit of a heuristic, '^)' occur commonly in Emscripten generated regexes i += 1 elif ch == '(': if rcount(s, '\\', i - 1) % 2 == 0: @@ -122,7 +123,9 @@ def parse_parens(s): # Valid characters in Emscripten outputted JS content (in reality valid character set is much more complex, but do not need that here) def is_javascript_symbol_char(ch): i = ord(ch) - return (i >= 97 and i <= 122) or (i >= 65 and i <= 90) or (i >= 48 and i <= 57) or i == 36 or i == 95 # a-z, A-Z, 0-9, $, _ + return ( + (i >= 97 and i <= 122) or (i >= 65 and i <= 90) or (i >= 48 and i <= 57) or i == 36 or i == 95 + ) # a-z, A-Z, 0-9, $, _ def cxxfilt(): @@ -167,13 +170,15 @@ def merge_entry_to_existing(existing_data, new_entry, total_source_set_size): existing_data[name] = { 'lines': ex['lines'] + new_entry['lines'], 'bytes': ex['bytes'] + new_entry['bytes'], - 'demangled_name': ex['demangled_name'] if 'demangled_name' in ex else (new_entry['demangled_name'] if 'demangled_name' in new_entry else new_entry['minified_name']), + 'demangled_name': ex['demangled_name'] + if 'demangled_name' in ex + else (new_entry['demangled_name'] if 'demangled_name' in new_entry else new_entry['minified_name']), 'minified_name': ex['minified_name'], 'unminified_name': ex['unminified_name'], 'function_parameters': ex['function_parameters'], 'type': ex['type'], 'percentage': (ex['bytes'] + new_entry['bytes']) * 100.0 / total_source_set_size, - 'num_times_occurs': num_times_occurs_1 + num_times_occurs_2 + 'num_times_occurs': num_times_occurs_1 + num_times_occurs_2, } else: existing_data[name] = new_entry @@ -196,9 +201,11 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s parse_pos = 0 prev_end_pos = 0 file_len = len(file_contents) - func_regex = re.compile(r'function\s+([\w$]+)\s*\(([\w\s$,]*?)\)\s*{') # Search for "function foo (param1, param2, ..., paranN) {" - var_block_regex = re.compile(r'var\s+(\w+)\s*=\s*([{\[\(])') # Search for "var foo = {" - var_regex = re.compile(r'var\s+([\w]+)\s*=\s*[\w\s,]*?;') # Search for "var foo = .... ;" + func_regex = re.compile( + r'function\s+([\w$]+)\s*\(([\w\s$,]*?)\)\s*{' + ) # Search for "function foo (param1, param2, ..., paranN) {" + var_block_regex = re.compile(r'var\s+(\w+)\s*=\s*([{\[\(])') # Search for "var foo = {" + var_regex = re.compile(r'var\s+([\w]+)\s*=\s*[\w\s,]*?;') # Search for "var foo = .... ;" unaccounted_bytes = 0 unaccounted_lines = 0 @@ -253,13 +260,15 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s # find starting and ending braces { } for the function start_brace = file_contents.find('{', func_pos) if start_brace < 0: - break # Must be at the end of file + break # Must be at the end of file if start_brace not in brace_map: - print('Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse function start brace, skipping.') + print( + 'Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse function start brace, skipping.' + ) continue end_brace = brace_map[start_brace] if end_brace < 0: - break # Must be at the end of file + break # Must be at the end of file num_bytes = end_brace + 1 - func_pos num_lines = file_contents.count('\n', func_pos, end_brace) + 1 @@ -279,21 +288,23 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s 'unminified_name': unminified_name, 'function_parameters': function_parameters, 'type': function_type, - 'percentage': num_bytes * 100.0 / total_source_set_size + 'percentage': num_bytes * 100.0 / total_source_set_size, } - else: # This is a variable + else: # This is a variable var_block_match = var_block_regex.match(file_contents[var_pos:]) if var_block_match: # find starting and ending braces { } for the var start_brace = file_contents.find(var_block_match.group(2), var_pos) if start_brace < 0: - break # Must be at the end of file + break # Must be at the end of file if start_brace not in brace_map: - print('Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse variable start brace, skipping.') + print( + 'Warning: ' + idx_to_line_col(file_contents, start_brace) + ' cannot parse variable start brace, skipping.' + ) continue end_brace = brace_map[start_brace] if end_brace < 0: - break # Must be at the end of file + break # Must be at the end of file minified_name = var_block_match.group(1) else: start_brace = var_pos @@ -305,7 +316,12 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s # Special case ignore the 'var wasmExports = (function(global, env, buffer) { 'use asm'; ... }; ' variable that contains all the asm.js code. # Ignoring this variable lets all the asm.js code be trated as functions in this parser, instead of assigning them to the asm variable. - if file_contents[start_brace] == '(' and ("'use asm'" in file_contents[var_pos:end_brace] or '"use asm"' in file_contents[var_pos:end_brace] or "'almost asm'" in file_contents[var_pos:end_brace] or '"almost asm"' in file_contents[var_pos:end_brace]): + if file_contents[start_brace] == '(' and ( + "'use asm'" in file_contents[var_pos:end_brace] + or '"use asm"' in file_contents[var_pos:end_brace] + or "'almost asm'" in file_contents[var_pos:end_brace] + or '"almost asm"' in file_contents[var_pos:end_brace] + ): continue num_bytes = end_brace + 1 - var_pos @@ -325,12 +341,12 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s 'unminified_name': unminified_name, 'function_parameters': '', 'type': var_type, - 'percentage': num_bytes * 100.0 / total_source_set_size + 'percentage': num_bytes * 100.0 / total_source_set_size, } if options.list_unaccounted: if diffing_two_data_sets: - unaccounted_name = '$unaccounted_js_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable + unaccounted_name = '$unaccounted_js_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable else: unaccounted_name = '$unaccounted_js_content_in("' + os.path.basename(filename) + '")' unaccounted_entry = { @@ -340,7 +356,7 @@ def analyze_javascript_file_contents(filename, file_contents, total_source_set_s 'unminified_name': unaccounted_name, 'function_parameters': '', 'type': '[UNKN]', - 'percentage': unaccounted_bytes * 100.0 / total_source_set_size + 'percentage': unaccounted_bytes * 100.0 / total_source_set_size, } merge_entry_to_existing(data, unaccounted_entry, total_source_set_size) @@ -377,7 +393,9 @@ def analyze_html_file(filename, total_source_set_size, symbol_map=None): if script_pos > parse_pos: unaccounted_bytes += script_pos - parse_pos unaccounted_lines += file_contents.count('\n', parse_pos, script_pos) + 1 - data_set = analyze_javascript_file_contents(filename, file_contents[script_pos:script_end_pos], total_source_set_size, symbol_map) + data_set = analyze_javascript_file_contents( + filename, file_contents[script_pos:script_end_pos], total_source_set_size, symbol_map + ) merge_to_data_set(data, data_set, total_source_set_size) parse_pos = script_end_pos @@ -387,7 +405,7 @@ def analyze_html_file(filename, total_source_set_size, symbol_map=None): if options.list_unaccounted and unaccounted_bytes > 0: if diffing_two_data_sets: - unaccounted_name = '$unaccounted_html_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable + unaccounted_name = '$unaccounted_html_content' # If diffing two data sets, must make the names of the unaccounted content blocks be comparable else: unaccounted_name = '$unaccounted_html_content_in("' + os.path.basename(filename) + '")' unaccounted_entry = { @@ -397,7 +415,7 @@ def analyze_html_file(filename, total_source_set_size, symbol_map=None): 'unminified_name': unaccounted_name, 'function_parameters': '', 'type': 'HTML', - 'percentage': unaccounted_bytes * 100.0 / total_source_set_size + 'percentage': unaccounted_bytes * 100.0 / total_source_set_size, } merge_entry_to_existing(data, unaccounted_entry, total_source_set_size) @@ -424,8 +442,14 @@ def common_compare(data1, data2): commonbytediff += d2['bytes'] - d1['bytes'] linesword = 'more' if commonlinediff >= 0 else 'less' bytesword = 'more' if commonbytediff >= 0 else 'less' - print('set 2 has {} lines {} than set 1 in {} common functions'.format(abs(commonlinediff), linesword, len(commonfns))) - print('set 2 has {} bytes {} than set 1 in {} common functions'.format(str(abs(commonbytediff)), bytesword, len(commonfns))) + print( + 'set 2 has {} lines {} than set 1 in {} common functions'.format(abs(commonlinediff), linesword, len(commonfns)) + ) + print( + 'set 2 has {} bytes {} than set 1 in {} common functions'.format( + str(abs(commonbytediff)), bytesword, len(commonfns) + ) + ) def uniq_compare(data1, data2): @@ -452,7 +476,11 @@ def uniq_compare(data1, data2): countword = 'more' if uniqcountdiff >= 0 else 'less' linesword = 'more' if uniqlinediff >= 0 else 'less' bytesword = 'more' if uniqbytediff >= 0 else 'less' - print('set 2 has {} functions {} than set 1 overall (unique: {} vs {})'.format(abs(uniqcountdiff), countword, len(uniqfns2), len(uniqfns1))) + print( + 'set 2 has {} functions {} than set 1 overall (unique: {} vs {})'.format( + abs(uniqcountdiff), countword, len(uniqfns2), len(uniqfns1) + ) + ) print('set 2 has {} lines {} than set 1 overall in unique functions'.format(abs(uniqlinediff), linesword)) print('set 2 has {} bytes {} than set 1 overall in unique functions'.format(str(abs(uniqbytediff)), bytesword)) @@ -469,9 +497,13 @@ def simplify_cxx_name(name): DEM_RE.append(lambda s: string_m.sub(r'std::string', s)) vec_m = re.compile(r'std::__2::vector<([^,]+), std::__2::allocator<\1\s*> >') DEM_RE.append(lambda s: vec_m.sub(r'std::vector<\1>', s)) - unordered_map_m = re.compile(r'std::__2::unordered_map<([^,]+), ([^,]+), std::__2::hash<\1\s*>, std::__2::equal_to<\1\s*>, std::__2::allocator > >') + unordered_map_m = re.compile( + r'std::__2::unordered_map<([^,]+), ([^,]+), std::__2::hash<\1\s*>, std::__2::equal_to<\1\s*>, std::__2::allocator > >' + ) DEM_RE.append(lambda s: unordered_map_m.sub(r'std::unordered_map<\1, \2>', s)) - sort_m = re.compile(r'std::__2::__sort&, \1\*>\(\1\*, \1\*, std::__2::__less<\1, \1\s*>&\)') + sort_m = re.compile( + r'std::__2::__sort&, \1\*>\(\1\*, \1\*, std::__2::__less<\1, \1\s*>&\)' + ) DEM_RE.append(lambda s: sort_m.sub(r'std::sort(\1*, \1*)', s)) DEM_RE.append(lambda s: s.replace('std::__2::', 'std::')) @@ -483,7 +515,7 @@ def simplify_cxx_name(name): # 'foo(int, float)' -> 'foo' def function_args_removed(s): if '(' in s: - return s[:s.find('(')] + return s[: s.find('(')] else: return s @@ -491,7 +523,7 @@ def function_args_removed(s): # 'foo(int, float)' -> 'int, float)' def function_args_part(s): if '(' in s: - return s[s.find('(') + 1:] + return s[s.find('(') + 1 :] else: return '' @@ -504,6 +536,7 @@ def sort_key_py2(key_value): # def sort_key_py3(key, value): # return value[options.sort] + def print_symbol_info(data, total_source_set_size): data = list(data.items()) data.sort(key=sort_key_py2, reverse=not options.sort_ascending) @@ -566,7 +599,7 @@ def print_symbol_info(data, total_source_set_size): i = 0 while i + 1 < len(print_name): if print_name[i] == print_name[i + 1]: - print_name = print_name[:i] + print_name[i + 1:] + print_name = print_name[:i] + print_name[i + 1 :] continue n1 = function_args_removed(print_name[i]) n2 = function_args_removed(print_name[i + 1]) @@ -574,22 +607,33 @@ def print_symbol_info(data, total_source_set_size): args2 = function_args_part(print_name[i + 1]) if n1 == n2 and (not args1 or not args2): if not args1: - print_name = print_name[:i] + print_name[i + 1:] + print_name = print_name[:i] + print_name[i + 1 :] else: - print_name = print_name[:i + 1] + print_name[i + 2:] + print_name = print_name[: i + 1] + print_name[i + 2 :] continue i += 1 print_name = ' ; '.join(print_name) if 'num_times_occurs' in e: print_name = '[' + str(e['num_times_occurs']) + ' times] ' + print_name - delta_string = ' %+8d (%+6.2f%%)' % (e['bytes'] - e['prev_bytes'], e['percentage'] - e['prev_percentage']) if diffing_two_data_sets else '' - print('%6d lines %7s (%5.2f%%) %s: %8s %s' % (e['lines'], str(e['bytes']), e['percentage'], delta_string, e['type'], print_name)) + delta_string = ( + ' %+8d (%+6.2f%%)' % (e['bytes'] - e['prev_bytes'], e['percentage'] - e['prev_percentage']) + if diffing_two_data_sets + else '' + ) + print( + '%6d lines %7s (%5.2f%%) %s: %8s %s' + % (e['lines'], str(e['bytes']), e['percentage'], delta_string, e['type'], print_name) + ) total_size += e['bytes'] if total_size < total_source_set_size: - print('Total size of printed functions: ' + str(total_size) + ' bytes. (%.2f%% of all symbols)' % (total_size * 100.0 / total_source_set_size)) + print( + 'Total size of printed functions: ' + + str(total_size) + + ' bytes. (%.2f%% of all symbols)' % (total_size * 100.0 / total_source_set_size) + ) else: print('Total size of printed functions: ' + str(total_size) + ' bytes.') @@ -746,68 +790,148 @@ def main(): usage_str = "emdump.py prints out statistics about compiled code sizes.\npython emdump.py --file a.js [--file2 b.js]" parser = argparse.ArgumentParser(usage=usage_str) - parser.add_argument('--file', dest='file', default=[], nargs='*', - help='Specifies the compiled JavaScript build file to analyze.') - - parser.add_argument('--file1', dest='file1', default=[], nargs='*', - help='Specifies the compiled JavaScript build file to analyze.') - - parser.add_argument('--symbol-map', dest='symbol_map', default='', - help='Specifies a filename to the symbol map file that can be used to unminify function and variable names.') - - parser.add_argument('--file2', dest='file2', default=[], nargs='*', - help='Specifies a second compiled JavaScript build file to analyze.') - - parser.add_argument('--symbol-map2', dest='symbol_map2', default='', - help='Specifies a filename to a second symbol map file that will be used to unminify function and variable names of file2.') - - parser.add_argument('--list-unaccounted', dest='list_unaccounted', type=int, default=1, - help='Pass --list-unaccounted=0 to skip listing a summary entry of unaccounted content') - - parser.add_argument('--dump-unaccounted-larger-than', dest='dump_unaccounted_larger_than', type=int, default=-1, - help='If an integer value >= 0 is specified, all unaccounted strings of content longer than the given value will be printed out to the console.\n(Note that it is common to have several unaccounted blocks, this is provided for curiosity/debugging/optimization ideas)') - - parser.add_argument('--only-unique-1', dest='only_unique_1', action='store_true', default=False, - help='If two data sets are specified, prints out only the symbols that are present in set 1, but not in set 2') - - parser.add_argument('--only-unique-2', dest='only_unique_2', action='store_true', default=False, - help='If two data sets are specified, prints out only the symbols that are present in set 2, but not in set 1') - - parser.add_argument('--only-common', dest='only_common', action='store_true', default=False, - help='If two data sets are specified, prints out only the symbols that are common to both data sets') - - parser.add_argument('--only-changes', dest='only_changes', action='store_true', default=False, - help='If two data sets are specified, prints out only the symbols that have changed size or are added/removed') - - parser.add_argument('--only-summarize', dest='only_summarize', action='store_true', default=False, - help='If specified, detailed information about each symbol is not printed, but only summary data is shown.') - - parser.add_argument('--filter-name', dest='filter_name', default='', - help='Only prints out information about symbols that contain the given filter substring in their demangled names. The filtering is always performed in lower case.') - - parser.add_argument('--filter-size', dest='filter_size', type=int, default=0, - help='Only prints out information about symbols that are (or were) larger than the given amount of bytes.') - - parser.add_argument('--sort', dest='sort', default='bytes', - help='Specifies the data column to sort output by. Possible values are: lines, bytes, delta, abs_delta, type, minified, unminified, demangled') - - parser.add_argument('--print-format', dest='print_format', default='DM', - help='Specifies the naming format for the symbols. Possible options are one of: m, u, d, du, dm, um, dum. Here "m" denotes minified, "u" denotes unminified, and "d" denotes demangled. Specify any combination of the characters in upper case to print out function parameters.\nDefault: DM.') - - parser.add_argument('--sort-ascending', dest='sort_ascending', action='store_true', default=False, - help='If true, reverses the sorting order to be ascending instead of default descending.') - - parser.add_argument('--simplify-cxx', dest='simplify_cxx', action='store_true', default=False, - help='Simplify C++ STL types as much as possible in the output') - - parser.add_argument('--group-templates', dest='group_templates', action='store_true', default=False, - help='Group/collapse all C++ templates with Foo and Foo to generic Foo') + parser.add_argument( + '--file', dest='file', default=[], nargs='*', help='Specifies the compiled JavaScript build file to analyze.' + ) + + parser.add_argument( + '--file1', dest='file1', default=[], nargs='*', help='Specifies the compiled JavaScript build file to analyze.' + ) + + parser.add_argument( + '--symbol-map', + dest='symbol_map', + default='', + help='Specifies a filename to the symbol map file that can be used to unminify function and variable names.', + ) + + parser.add_argument( + '--file2', dest='file2', default=[], nargs='*', help='Specifies a second compiled JavaScript build file to analyze.' + ) + + parser.add_argument( + '--symbol-map2', + dest='symbol_map2', + default='', + help='Specifies a filename to a second symbol map file that will be used to unminify function and variable names of file2.', + ) + + parser.add_argument( + '--list-unaccounted', + dest='list_unaccounted', + type=int, + default=1, + help='Pass --list-unaccounted=0 to skip listing a summary entry of unaccounted content', + ) + + parser.add_argument( + '--dump-unaccounted-larger-than', + dest='dump_unaccounted_larger_than', + type=int, + default=-1, + help='If an integer value >= 0 is specified, all unaccounted strings of content longer than the given value will be printed out to the console.\n(Note that it is common to have several unaccounted blocks, this is provided for curiosity/debugging/optimization ideas)', + ) + + parser.add_argument( + '--only-unique-1', + dest='only_unique_1', + action='store_true', + default=False, + help='If two data sets are specified, prints out only the symbols that are present in set 1, but not in set 2', + ) + + parser.add_argument( + '--only-unique-2', + dest='only_unique_2', + action='store_true', + default=False, + help='If two data sets are specified, prints out only the symbols that are present in set 2, but not in set 1', + ) + + parser.add_argument( + '--only-common', + dest='only_common', + action='store_true', + default=False, + help='If two data sets are specified, prints out only the symbols that are common to both data sets', + ) + + parser.add_argument( + '--only-changes', + dest='only_changes', + action='store_true', + default=False, + help='If two data sets are specified, prints out only the symbols that have changed size or are added/removed', + ) + + parser.add_argument( + '--only-summarize', + dest='only_summarize', + action='store_true', + default=False, + help='If specified, detailed information about each symbol is not printed, but only summary data is shown.', + ) + + parser.add_argument( + '--filter-name', + dest='filter_name', + default='', + help='Only prints out information about symbols that contain the given filter substring in their demangled names. The filtering is always performed in lower case.', + ) + + parser.add_argument( + '--filter-size', + dest='filter_size', + type=int, + default=0, + help='Only prints out information about symbols that are (or were) larger than the given amount of bytes.', + ) + + parser.add_argument( + '--sort', + dest='sort', + default='bytes', + help='Specifies the data column to sort output by. Possible values are: lines, bytes, delta, abs_delta, type, minified, unminified, demangled', + ) + + parser.add_argument( + '--print-format', + dest='print_format', + default='DM', + help='Specifies the naming format for the symbols. Possible options are one of: m, u, d, du, dm, um, dum. Here "m" denotes minified, "u" denotes unminified, and "d" denotes demangled. Specify any combination of the characters in upper case to print out function parameters.\nDefault: DM.', + ) + + parser.add_argument( + '--sort-ascending', + dest='sort_ascending', + action='store_true', + default=False, + help='If true, reverses the sorting order to be ascending instead of default descending.', + ) + + parser.add_argument( + '--simplify-cxx', + dest='simplify_cxx', + action='store_true', + default=False, + help='Simplify C++ STL types as much as possible in the output', + ) + + parser.add_argument( + '--group-templates', + dest='group_templates', + action='store_true', + default=False, + help='Group/collapse all C++ templates with Foo and Foo to generic Foo', + ) options = parser.parse_args() options.file = options.file + options.file1 if not options.file: - print('Specify a set of JavaScript build output files to analyze with --file file1.js file2.js ... fileN.js.\nRun python emdump.py --help to see all options.') + print( + 'Specify a set of JavaScript build output files to analyze with --file file1.js file2.js ... fileN.js.\nRun python emdump.py --help to see all options.' + ) return 1 options.filter_name = options.filter_name.lower() @@ -815,15 +939,21 @@ def main(): diffing_two_data_sets = len(options.file2) > 0 if not diffing_two_data_sets: if options.only_unique_1: - print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-1 option!') + print( + 'Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-1 option!' + ) sys.exit(1) if options.only_unique_2: - print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-2 option!') + print( + 'Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-unique-symbols-in-set-2 option!' + ) sys.exit(1) if options.only_common: - print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-common-symbols option!') + print( + 'Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --only-common-symbols option!' + ) sys.exit(1) # Validate column sorting input: @@ -839,7 +969,10 @@ def main(): options.sort = 'demangled_name' if 'delta' in options.sort and not diffing_two_data_sets: - print('Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --sort=' + options.sort) + print( + 'Error: Must specify two data sets with --file a.js b.js c.js --file2 d.js e.js f.js to diff in order to use --sort=' + + options.sort + ) sys.exit(1) # Autoguess .symbols file location based on default Emscripten build output, to save the need to type it out in the common case @@ -873,7 +1006,10 @@ def main(): if not options.only_summarize: print_symbol_info(diffed_data, set2_size) print('') - print('set 2 is %d bytes, which is %+.2f%% %s than set 1 size (%d bytes)' % (set2_size, (set2_size - set1_size) * 100.0 / set2_size, 'more' if set2_size > set1_size else 'less', set1_size)) + print( + 'set 2 is %d bytes, which is %+.2f%% %s than set 1 size (%d bytes)' + % (set2_size, (set2_size - set1_size) * 100.0 / set2_size, 'more' if set2_size > set1_size else 'less', set1_size) + ) uniq_compare(data1, data2) common_compare(data1, data2) else: diff --git a/tools/emdwp.py b/tools/emdwp.py index fa21c0fd5664f..1fe4ae923c958 100755 --- a/tools/emdwp.py +++ b/tools/emdwp.py @@ -4,8 +4,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Wrapper script around `llvm-dwp`. -""" +"""Wrapper script around `llvm-dwp`.""" import sys import os diff --git a/tools/emnm.py b/tools/emnm.py index 6ad9a7899fd06..025b56f561fe7 100755 --- a/tools/emnm.py +++ b/tools/emnm.py @@ -4,8 +4,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Wrapper script around `llvm-nm`. -""" +"""Wrapper script around `llvm-nm`.""" import os import sys diff --git a/tools/emprofile.py b/tools/emprofile.py index 6b0259739a384..7ccdd2c1885df 100755 --- a/tools/emprofile.py +++ b/tools/emprofile.py @@ -55,7 +55,10 @@ def create_profiling_graph(outfile): print('Failed to parse JSON file "' + f + '"!', file=sys.stderr) return 1 if len(all_results) == 0: - print(f'No profiler logs were found in path: ${profiler_logs_path}.\nTry setting the environment variable EMPROFILE=1 and run some emcc commands, then re-run "emprofile.py --graph".', file=sys.stderr) + print( + f'No profiler logs were found in path: ${profiler_logs_path}.\nTry setting the environment variable EMPROFILE=1 and run some emcc commands, then re-run "emprofile.py --graph".', + file=sys.stderr, + ) return 1 all_results.sort(key=lambda x: x['time']) @@ -63,7 +66,11 @@ def create_profiling_graph(outfile): emprofile_json_data = json.dumps(all_results, indent=2) html_file = outfile + '.html' - html_contents = Path(os.path.dirname(os.path.realpath(__file__)), 'toolchain_profiler.results_template.html').read_text().replace('{{{ emprofile_json_data }}}', emprofile_json_data) + html_contents = ( + Path(os.path.dirname(os.path.realpath(__file__)), 'toolchain_profiler.results_template.html') + .read_text() + .replace('{{{ emprofile_json_data }}}', emprofile_json_data) + ) Path(html_file).write_text(html_contents) print(f'Wrote "{html_file}"') return 0 diff --git a/tools/experimental/reproduceriter.py b/tools/experimental/reproduceriter.py index 2813a83b28896..d78077c9a5bdc 100644 --- a/tools/experimental/reproduceriter.py +++ b/tools/experimental/reproduceriter.py @@ -150,8 +150,14 @@ print(' ', fullname) with open(fullname) as fh: js = fh.read() - js = re.sub(r'document\.on(\w+) ?= ?([\w.$]+)', lambda m: 'Recorder.onEvent("' + m.group(1) + '", ' + m.group(2) + ')', js) - js = re.sub(r'''([\w.'"\[\]]+)\.addEventListener\(([\w,. $]+)\)''', lambda m: 'Recorder.addListener(' + m.group(1) + ', ' + m.group(2) + ')', js) + js = re.sub( + r'document\.on(\w+) ?= ?([\w.$]+)', lambda m: 'Recorder.onEvent("' + m.group(1) + '", ' + m.group(2) + ')', js + ) + js = re.sub( + r'''([\w.'"\[\]]+)\.addEventListener\(([\w,. $]+)\)''', + lambda m: 'Recorder.addListener(' + m.group(1) + ', ' + m.group(2) + ')', + js, + ) Path(fullname).write_text(js) # Add our boilerplate @@ -160,11 +166,15 @@ with open(os.path.join(out_dir, first_js), 'w') as fh1: fh1.write( - (Path(os.path.dirname(os.path.dirname(__file__)), 'src', 'headless.js').read_text() % ( - window_location, window_location.split('?')[-1], on_idle or 'null', dirs_to_drop - ) if shell else '') + - Path(os.path.dirname(__file__), 'reproduceriter.js').read_text() + - Path(in_dir, first_js).read_text() + ('\nwindow.runEventLoop();\n' if shell else '') + ( + Path(os.path.dirname(os.path.dirname(__file__)), 'src', 'headless.js').read_text() + % (window_location, window_location.split('?')[-1], on_idle or 'null', dirs_to_drop) + if shell + else '' + ) + + Path(os.path.dirname(__file__), 'reproduceriter.js').read_text() + + Path(in_dir, first_js).read_text() + + ('\nwindow.runEventLoop();\n' if shell else '') ) print('done!') diff --git a/tools/extract_metadata.py b/tools/extract_metadata.py index a5159f24a1da8..d15e8f0d982c0 100644 --- a/tools/extract_metadata.py +++ b/tools/extract_metadata.py @@ -96,7 +96,14 @@ def parse_function_for_memory_inits(module, func_index, offset_map): module.read_type() elif opcode in (OpCode.I32_CONST, OpCode.I64_CONST): const_values.append(module.read_sleb()) - elif opcode in (OpCode.GLOBAL_SET, OpCode.BR, OpCode.GLOBAL_GET, OpCode.LOCAL_SET, OpCode.LOCAL_GET, OpCode.LOCAL_TEE): + elif opcode in ( + OpCode.GLOBAL_SET, + OpCode.BR, + OpCode.GLOBAL_GET, + OpCode.LOCAL_SET, + OpCode.LOCAL_GET, + OpCode.LOCAL_TEE, + ): module.read_uleb() elif opcode == OpCode.CALL: call_targets.append(module.read_uleb()) @@ -110,17 +117,21 @@ def parse_function_for_memory_inits(module, func_index, offset_map): memory = module.read_uleb() assert memory == 0 elif opcode == MemoryOpCode.MEMORY_FILL: - memory = module.read_uleb() # noqa + memory = module.read_uleb() # noqa assert memory == 0 elif opcode == MemoryOpCode.MEMORY_DROP: - segment = module.read_uleb() # noqa + segment = module.read_uleb() # noqa else: assert False, "unknown: %s" % opcode elif opcode == OpCode.ATOMIC_PREFIX: opcode = AtomicOpCode(module.read_byte()) - if opcode in (AtomicOpCode.ATOMIC_I32_RMW_CMPXCHG, AtomicOpCode.ATOMIC_I32_STORE, - AtomicOpCode.ATOMIC_NOTIFY, AtomicOpCode.ATOMIC_WAIT32, - AtomicOpCode.ATOMIC_WAIT64): + if opcode in ( + AtomicOpCode.ATOMIC_I32_RMW_CMPXCHG, + AtomicOpCode.ATOMIC_I32_STORE, + AtomicOpCode.ATOMIC_NOTIFY, + AtomicOpCode.ATOMIC_WAIT32, + AtomicOpCode.ATOMIC_WAIT64, + ): module.read_uleb() module.read_uleb() else: @@ -128,8 +139,8 @@ def parse_function_for_memory_inits(module, func_index, offset_map): elif opcode == OpCode.BR_TABLE: count = module.read_uleb() for _ in range(count): - depth = module.read_uleb() # noqa - default = module.read_uleb() # noqa + depth = module.read_uleb() # noqa + default = module.read_uleb() # noqa else: assert False, "unknown: %s" % opcode @@ -151,7 +162,7 @@ def get_passive_segment_offsets(module): def to_unsigned(val): if val < 0: - return val & ((2 ** 32) - 1) + return val & ((2**32) - 1) else: return val diff --git a/tools/feature_matrix.py b/tools/feature_matrix.py index 61c9399dd1aad..07b5793de79e3 100644 --- a/tools/feature_matrix.py +++ b/tools/feature_matrix.py @@ -133,17 +133,16 @@ def enable_feature(feature, reason, override=False): if name in user_settings: # If the user explicitly chose an older version we issue a warning. diagnostics.warning( - 'compatibility', - f'{name}={user_settings[name]} is not compatible with {reason} ' - f'({min_version} or above required)') + 'compatibility', + f'{name}={user_settings[name]} is not compatible with {reason} ' f'({min_version} or above required)', + ) else: # Otherwise we bump the minimum version to accommodate the feature. setattr(settings, name, min_version) def disable_feature(feature): - """Allow the user to disable a feature that would otherwise be on by default. - """ + """Allow the user to disable a feature that would otherwise be on by default.""" disable_override_features.add(feature) diff --git a/tools/file_packager.py b/tools/file_packager.py index f2e2c7729fd01..8f6155abf2895 100755 --- a/tools/file_packager.py +++ b/tools/file_packager.py @@ -157,8 +157,7 @@ def has_hidden_attribute(filepath): return False try: - attrs = ctypes.windll.kernel32.GetFileAttributesW( - u'%s' % filepath) + attrs = ctypes.windll.kernel32.GetFileAttributesW('%s' % filepath) assert attrs != -1 result = bool(attrs & 2) except Exception: @@ -192,20 +191,16 @@ def add(mode, rootpathsrc, rootpathdst): walked.append(fullname) new_dirnames.append(name) elif DEBUG: - err('Skipping directory "%s" from inclusion in the emscripten ' - 'virtual file system.' % fullname) + err('Skipping directory "%s" from inclusion in the emscripten ' 'virtual file system.' % fullname) for name in filenames: fullname = os.path.join(dirpath, name) if not should_ignore(fullname): walked.append(fullname) # Convert source filename relative to root directory of target FS. - dstpath = os.path.join(rootpathdst, - os.path.relpath(fullname, rootpathsrc)) - new_data_files.append(DataFile(srcpath=fullname, dstpath=dstpath, - mode=mode, explicit_dst_path=True)) + dstpath = os.path.join(rootpathdst, os.path.relpath(fullname, rootpathsrc)) + new_data_files.append(DataFile(srcpath=fullname, dstpath=dstpath, mode=mode, explicit_dst_path=True)) elif DEBUG: - err('Skipping file "%s" from inclusion in the emscripten ' - 'virtual file system.' % fullname) + err('Skipping file "%s" from inclusion in the emscripten ' 'virtual file system.' % fullname) dirnames.clear() dirnames.extend(new_dirnames) @@ -242,11 +237,13 @@ def escape(c): def to_c_symbol(filename, used): """Convert a filename (python string) to a legal C symbols, avoiding collisions.""" + def escape(c): - if c.isalnum(): - return c - else: - return '_' + if c.isalnum(): + return c + else: + return '_' + c_symbol = ''.join(escape(c) for c in filename) # Handle collisions if c_symbol in used: @@ -278,7 +275,8 @@ def generate_object_file(data_files): size = os.path.getsize(f.srcpath) dstpath = to_asm_string(f.dstpath) srcpath = utils.normalize_path(f.srcpath) - out.write(dedent(f''' + out.write( + dedent(f''' .section .rodata.{f.c_symbol_name},"",@ # The name of file @@ -290,7 +288,8 @@ def generate_object_file(data_files): {f.c_symbol_name}: .incbin "{srcpath}" .size {f.c_symbol_name}, {size} - ''')) + ''') + ) if options.wasm64: align = 3 @@ -300,7 +299,8 @@ def generate_object_file(data_files): align = 2 ptr_type = 'i32' bits = 32 - out.write(dedent(f''' + out.write( + dedent(f''' .functype _emscripten_fs_load_embedded_files ({ptr_type}) -> () .section .text,"",@ init_file_data: @@ -323,27 +323,35 @@ def generate_object_file(data_files): .section .rodata.__emscripten_embedded_file_data,"",@ __emscripten_embedded_file_data: .p2align {align} - ''')) + ''') + ) for f in embed_files: # The `.dc.a` directive gives us a pointer (address) sized entry. # See https://sourceware.org/binutils/docs/as/Dc.html - out.write(dedent(f'''\ + out.write( + dedent( + f'''\ .p2align %s .dc.a {f.c_symbol_name}_name .p2align %s .int32 {os.path.getsize(f.srcpath)} .p2align %s .dc.a {f.c_symbol_name} - ''' % (align, align, align))) + ''' + % (align, align, align) + ) + ) ptr_size = 4 elem_size = (2 * ptr_size) + 4 total_size = len(embed_files) * elem_size + 4 - out.write(dedent(f'''\ + out.write( + dedent(f'''\ .dc.a 0 .size __emscripten_embedded_file_data, {total_size} - ''')) + ''') + ) cmd = [shared.EMCC, '-c', asm_file, '-o', options.obj_output] if options.wasm64: target = 'wasm64-unknown-emscripten' @@ -426,7 +434,7 @@ def main(): # noqa: C901, PLR0912, PLR0915 options.quiet = True elif arg.startswith('--plugin'): plugin = utils.read_file(arg.split('=', 1)[1]) - eval(plugin) # should append itself to plugins + eval(plugin) # should append itself to plugins leading = '' elif leading == 'preload' or leading == 'embed': mode = leading @@ -435,17 +443,16 @@ def main(): # noqa: C901, PLR0912, PLR0915 at_position = arg.replace('@@', '__').find('@') # '@@' in input string means there is an actual @ character, a single '@' # means the 'src@dst' notation. - uses_at_notation = (at_position != -1) + uses_at_notation = at_position != -1 if uses_at_notation: - srcpath = arg[0:at_position].replace('@@', '@') # split around the @ - dstpath = arg[at_position + 1:].replace('@@', '@') + srcpath = arg[0:at_position].replace('@@', '@') # split around the @ + dstpath = arg[at_position + 1 :].replace('@@', '@') else: # Use source path as destination path. srcpath = dstpath = arg.replace('@@', '@') if os.path.isfile(srcpath) or os.path.isdir(srcpath): - data_files.append(DataFile(srcpath=srcpath, dstpath=dstpath, mode=mode, - explicit_dst_path=uses_at_notation)) + data_files.append(DataFile(srcpath=srcpath, dstpath=dstpath, mode=mode, explicit_dst_path=uses_at_notation)) else: err('error: ' + arg + ' does not exist') return 1 @@ -460,13 +467,14 @@ def main(): # noqa: C901, PLR0912, PLR0915 if options.separate_metadata: if not options.has_preloaded or not options.jsoutput: - err('cannot separate-metadata without both --preloaded files ' - 'and a specified --js-output') + err('cannot separate-metadata without both --preloaded files ' 'and a specified --js-output') return 1 if not options.from_emcc and not options.quiet: - err('Remember to build the main file with `-sFORCE_FILESYSTEM` ' - 'so that it includes support for loading this file package') + err( + 'Remember to build the main file with `-sFORCE_FILESYSTEM` ' + 'so that it includes support for loading this file package' + ) if options.jsoutput and os.path.abspath(options.jsoutput) == os.path.abspath(data_target): err('error: TARGET should not be the same value of --js-output') @@ -480,8 +488,7 @@ def main(): # noqa: C901, PLR0912, PLR0915 else: walked.append(file_.srcpath) new_data_files.append(file_) - data_files = [file_ for file_ in new_data_files - if not os.path.isdir(file_.srcpath)] + data_files = [file_ for file_ in new_data_files if not os.path.isdir(file_.srcpath)] if len(data_files) == 0: err('Nothing to do!') sys.exit(1) @@ -503,19 +510,23 @@ def main(): # noqa: C901, PLR0912, PLR0915 if DEBUG: err(path, abspath, curr_abspath) if not abspath.startswith(curr_abspath): - err('Error: Embedding "%s" which is not contained within the current directory ' - '"%s". This is invalid since the current directory becomes the ' - 'root that the generated code will see. To include files outside of the current ' - 'working directory you can use the `--preload-file srcpath@dstpath` syntax to ' - 'explicitly specify the target location.' % (path, curr_abspath)) + err( + 'Error: Embedding "%s" which is not contained within the current directory ' + '"%s". This is invalid since the current directory becomes the ' + 'root that the generated code will see. To include files outside of the current ' + 'working directory you can use the `--preload-file srcpath@dstpath` syntax to ' + 'explicitly specify the target location.' % (path, curr_abspath) + ) sys.exit(1) - file_.dstpath = abspath[len(curr_abspath) + 1:] + file_.dstpath = abspath[len(curr_abspath) + 1 :] if os.path.isabs(path): - err('Warning: Embedding an absolute file/directory name "%s" to the ' - 'virtual filesystem. The file will be made available in the ' - 'relative path "%s". You can use the `--preload-file srcpath@dstpath` ' - 'syntax to explicitly specify the target location the absolute source ' - 'path should be directed to.' % (path, file_.dstpath)) + err( + 'Warning: Embedding an absolute file/directory name "%s" to the ' + 'virtual filesystem. The file will be made available in the ' + 'relative path "%s". You can use the `--preload-file srcpath@dstpath` ' + 'syntax to explicitly specify the target location the absolute source ' + 'path should be directed to.' % (path, file_.dstpath) + ) for file_ in data_files: # name in the filesystem, native and emulated @@ -527,7 +538,7 @@ def main(): # noqa: C901, PLR0912, PLR0915 # make destination path always relative to the root file_.dstpath = posixpath.normpath(os.path.join('/', file_.dstpath)) if DEBUG: - err('Packaging file "%s" to VFS in path "%s".' % (file_.srcpath, file_.dstpath)) + err('Packaging file "%s" to VFS in path "%s".' % (file_.srcpath, file_.dstpath)) # Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/) seen = set() @@ -633,14 +644,16 @@ def generate_js(data_target, data_files, metadata): partial_dirs = [] for file_ in data_files: dirname = os.path.dirname(file_.dstpath) - dirname = dirname.lstrip('/') # absolute paths start with '/', remove that + dirname = dirname.lstrip('/') # absolute paths start with '/', remove that if dirname != '': parts = dirname.split('/') for i in range(len(parts)): - partial = '/'.join(parts[:i + 1]) + partial = '/'.join(parts[: i + 1]) if partial not in partial_dirs: - code += ('''Module['FS_createPath'](%s, %s, true, true);\n''' - % (json.dumps('/' + '/'.join(parts[:i])), json.dumps(parts[i]))) + code += '''Module['FS_createPath'](%s, %s, true, true);\n''' % ( + json.dumps('/' + '/'.join(parts[:i])), + json.dumps(parts[i]), + ) partial_dirs.append(partial) if options.has_preloaded: @@ -653,15 +666,17 @@ def generate_js(data_target, data_files, metadata): curr = utils.read_binary(file_.srcpath) file_.data_end = start + len(curr) if AV_WORKAROUND: - curr += '\x00' + curr += '\x00' start += len(curr) data.write(curr) if start > 256 * 1024 * 1024: - err('warning: file packager is creating an asset bundle of %d MB. ' - 'this is very large, and browsers might have trouble loading it. ' - 'see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/' - % (start / (1024 * 1024))) + err( + 'warning: file packager is creating an asset bundle of %d MB. ' + 'this is very large, and browsers might have trouble loading it. ' + 'see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/' + % (start / (1024 * 1024)) + ) create_preloaded = ''' Module['FS_createPreloadedFile'](this.name, null, byteArray, true, true, @@ -707,7 +722,9 @@ def generate_js(data_target, data_files, metadata): }\n''' % (create_preloaded if options.use_preload_plugins else create_data) if options.has_embedded and not options.obj_output: - err('--obj-output is recommended when using --embed. This outputs an object file for linking directly into your application is more efficient than JS encoding') + err( + '--obj-output is recommended when using --embed. This outputs an object file for linking directly into your application is more efficient than JS encoding' + ) for counter, file_ in enumerate(data_files): filename = file_.dstpath @@ -719,8 +736,11 @@ def generate_js(data_target, data_files, metadata): data = base64_encode(utils.read_binary(file_.srcpath)) code += " var fileData%d = '%s';\n" % (counter, data) # canOwn this data in the filesystem (i.e. there is no need to create a copy in the FS layer). - code += (" Module['FS_createDataFile']('%s', '%s', atob(fileData%d), true, true, true);\n" - % (dirname, basename, counter)) + code += " Module['FS_createDataFile']('%s', '%s', atob(fileData%d), true, true, true);\n" % ( + dirname, + basename, + counter, + ) elif file_.mode == 'preload': # Preload metadata_el = { @@ -744,21 +764,25 @@ def generate_js(data_target, data_files, metadata): for (var i = 0; i < files.length; ++i) { DataRequest.prototype.requests[files[i].filename].onload(); }''' - use_data += (" Module['removeRunDependency']('datafile_%s');\n" - % js_manipulation.escape_for_js_string(data_target)) + use_data += " Module['removeRunDependency']('datafile_%s');\n" % js_manipulation.escape_for_js_string( + data_target + ) else: # LZ4FS usage temp = data_target + '.orig' shutil.move(data_target, temp) - meta = shared.run_js_tool(utils.path_from_root('tools/lz4-compress.mjs'), - [temp, data_target], stdout=PIPE) + meta = shared.run_js_tool(utils.path_from_root('tools/lz4-compress.mjs'), [temp, data_target], stdout=PIPE) os.unlink(temp) use_data = '''var compressedData = %s; compressedData['data'] = byteArray; assert(typeof Module['LZ4'] === 'object', 'LZ4 not present - was your app build with -sLZ4?'); Module['LZ4'].loadPackage({ 'metadata': metadata, 'compressedData': compressedData }, %s); - Module['removeRunDependency']('datafile_%s');''' % (meta, "true" if options.use_preload_plugins else "false", js_manipulation.escape_for_js_string(data_target)) + Module['removeRunDependency']('datafile_%s');''' % ( + meta, + "true" if options.use_preload_plugins else "false", + js_manipulation.escape_for_js_string(data_target), + ) package_name = data_target remote_package_size = os.path.getsize(package_name) @@ -773,7 +797,10 @@ def generate_js(data_target, data_files, metadata): } var PACKAGE_NAME = '%s'; var REMOTE_PACKAGE_BASE = '%s'; - var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % (js_manipulation.escape_for_js_string(data_target), js_manipulation.escape_for_js_string(remote_package_name)) + var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;\n''' % ( + js_manipulation.escape_for_js_string(data_target), + js_manipulation.escape_for_js_string(remote_package_name), + ) metadata['remote_package_size'] = remote_package_size ret += '''var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];\n''' @@ -784,11 +811,14 @@ def generate_js(data_target, data_files, metadata): package_uuid = 'sha256-' + hashlib.sha256(data).hexdigest() metadata['package_uuid'] = str(package_uuid) - code += r''' + code += ( + r''' var PACKAGE_UUID = metadata['package_uuid']; var IDB_RO = "readonly"; var IDB_RW = "readwrite"; - var DB_NAME = "''' + options.indexeddb_name + '''"; + var DB_NAME = "''' + + options.indexeddb_name + + '''"; var DB_VERSION = 1; var METADATA_STORE_NAME = 'METADATA'; var PACKAGE_STORE_NAME = 'PACKAGES'; @@ -941,6 +971,7 @@ def generate_js(data_target, data_files, metadata): getRequest.onerror = (error) => errback(error); } }\n''' + ) # add Node.js support code, if necessary node_support_code = '' diff --git a/tools/install.py b/tools/install.py index 368c669061a27..dde44cf92743c 100755 --- a/tools/install.py +++ b/tools/install.py @@ -17,7 +17,9 @@ import subprocess import sys -EXCLUDES = [os.path.normpath(x) for x in ''' +EXCLUDES = [ + os.path.normpath(x) + for x in ''' test/third_party tools/maint site @@ -27,7 +29,8 @@ cache cache.lock bootstrap.py -'''.split()] +'''.split() +] EXCLUDE_PATTERNS = ''' *.pyc @@ -85,8 +88,9 @@ def copy_emscripten(target): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('-v', '--verbose', action='store_true', help='verbose', - default=int(os.environ.get('EMCC_DEBUG', '0'))) + parser.add_argument( + '-v', '--verbose', action='store_true', help='verbose', default=int(os.environ.get('EMCC_DEBUG', '0')) + ) parser.add_argument('target', help='target directory') args = parser.parse_args() target = os.path.abspath(args.target) diff --git a/tools/js_manipulation.py b/tools/js_manipulation.py index 7790790b7c3f7..a36225549766b 100644 --- a/tools/js_manipulation.py +++ b/tools/js_manipulation.py @@ -41,18 +41,24 @@ def add_files_pre_js(pre_js_list, files_pre_js): # enabled pre = shared.get_temp_files().get('.js').name post = shared.get_temp_files().get('.js').name - utils.write_file(pre, ''' + utils.write_file( + pre, + ''' // All the pre-js content up to here must remain later on, we need to run // it. if (Module['$ww'] || (typeof ENVIRONMENT_IS_PTHREAD != 'undefined' && ENVIRONMENT_IS_PTHREAD)) Module['preRun'] = []; var necessaryPreJSTasks = Module['preRun'].slice(); - ''') - utils.write_file(post, ''' + ''', + ) + utils.write_file( + post, + ''' if (!Module['preRun']) throw 'Module.preRun should exist because file support used it; did a pre-js delete it?'; necessaryPreJSTasks.forEach((task) => { if (Module['preRun'].indexOf(task) < 0) throw 'All preRun tasks that exist before user pre-js code should remain after; did you replace Module or modify Module.preRun?'; }); - ''') + ''', + ) pre_js_list.insert(1, pre) pre_js_list.append(post) @@ -125,7 +131,7 @@ def make_dynCall(sig, args): def make_invoke(sig): - legal_sig = legalize_sig(sig) # TODO: do this in extcall, jscall? + legal_sig = legalize_sig(sig) # TODO: do this in extcall, jscall? args = ['index'] + ['a' + str(i) for i in range(1, len(legal_sig))] ret = 'return ' if sig[0] != 'v' else '' # For function that needs to return a genuine i64 (i.e. if legal_sig[0] is 'j') diff --git a/tools/js_optimizer.py b/tools/js_optimizer.py index d7a158b338ea3..ee0ba563f5a97 100755 --- a/tools/js_optimizer.py +++ b/tools/js_optimizer.py @@ -25,7 +25,9 @@ ACORN_OPTIMIZER = path_from_root('tools/acorn-optimizer.mjs') NUM_CHUNKS_PER_CORE = 3 -MIN_CHUNK_SIZE = int(os.environ.get('EMCC_JSOPT_MIN_CHUNK_SIZE') or 512 * 1024) # configuring this is just for debugging purposes +MIN_CHUNK_SIZE = int( + os.environ.get('EMCC_JSOPT_MIN_CHUNK_SIZE') or 512 * 1024 +) # configuring this is just for debugging purposes MAX_CHUNK_SIZE = int(os.environ.get('EMCC_JSOPT_MAX_CHUNK_SIZE') or 5 * 1024 * 1024) WINDOWS = sys.platform.startswith('win') @@ -80,7 +82,7 @@ def minify_shell(self, shell, minify_whitespace): # to minify all global names, we receive a dictionary back, which is then # used by the function processors - shell = shell.replace('0.0', '13371337') # avoid optimizer doing 0.0 => 0 + shell = shell.replace('0.0', '13371337') # avoid optimizer doing 0.0 => 0 # Find all globals in the JS functions code @@ -114,9 +116,7 @@ def minify_shell(self, shell, minify_whitespace): return code.replace('13371337', '0.0') def serialize(self): - return { - 'globals': self.globs - } + return {'globals': self.globs} start_funcs_marker = '// EMSCRIPTEN_START_FUNCS\n' @@ -145,7 +145,7 @@ def chunkify(funcs, chunk_size): if curr: chunks.append(curr) curr = None - return [''.join(func[1] for func in chunk) for chunk in chunks] # remove function names + return [''.join(func[1] for func in chunk) for chunk in chunks] # remove function names @ToolchainProfiler.profile_block('js_optimizer.run_on_file') @@ -156,14 +156,17 @@ def run_on_file(filename, passes, extra_info=None): js = utils.read_file(filename) if os.linesep != '\n': - js = js.replace(os.linesep, '\n') # we assume \n in the splitting code + js = js.replace(os.linesep, '\n') # we assume \n in the splitting code # Find markers start_funcs = js.find(start_funcs_marker) end_funcs = js.rfind(end_funcs_marker) if start_funcs < 0 or end_funcs < start_funcs: - shared.exit_with_error('invalid input file. Did not contain appropriate markers. (start_funcs: %s, end_funcs: %s' % (start_funcs, end_funcs)) + shared.exit_with_error( + 'invalid input file. Did not contain appropriate markers. (start_funcs: %s, end_funcs: %s' + % (start_funcs, end_funcs) + ) minify_globals = 'minifyNames' in passes if minify_globals: @@ -174,17 +177,17 @@ def run_on_file(filename, passes, extra_info=None): closure = 'closure' in passes if closure: - passes = [p for p in passes if p != 'closure'] # we will do it manually + passes = [p for p in passes if p != 'closure'] # we will do it manually cleanup = 'cleanup' in passes if cleanup: - passes = [p for p in passes if p != 'cleanup'] # we will do it manually + passes = [p for p in passes if p != 'cleanup'] # we will do it manually if not minify_globals: with ToolchainProfiler.profile_block('js_optimizer.no_minify_globals'): - pre = js[:start_funcs + len(start_funcs_marker)] - post = js[end_funcs + len(end_funcs_marker):] - js = js[start_funcs + len(start_funcs_marker):end_funcs] + pre = js[: start_funcs + len(start_funcs_marker)] + post = js[end_funcs + len(end_funcs_marker) :] + js = js[start_funcs + len(start_funcs_marker) : end_funcs] # can have Module[..] and inlining prevention code, push those to post finals = [] @@ -200,12 +203,16 @@ def process(line): else: with ToolchainProfiler.profile_block('js_optimizer.minify_globals'): # We need to split out the asm shell as well, for minification - pre = js[:start_asm + len(start_asm_marker)] + pre = js[: start_asm + len(start_asm_marker)] post = js[end_asm:] - asm_shell = js[start_asm + len(start_asm_marker):start_funcs + len(start_funcs_marker)] + ''' + asm_shell = ( + js[start_asm + len(start_asm_marker) : start_funcs + len(start_funcs_marker)] + + ''' EMSCRIPTEN_FUNCS(); -''' + js[end_funcs + len(end_funcs_marker):end_asm + len(end_asm_marker)] - js = js[start_funcs + len(start_funcs_marker):end_funcs] +''' + + js[end_funcs + len(end_funcs_marker) : end_asm + len(end_asm_marker)] + ) + js = js[start_funcs + len(start_funcs_marker) : end_funcs] # we assume there is a maximum of one new name per line minifier = Minifier(js) @@ -220,7 +227,9 @@ def check_symbol_mapping(p): return True passes = [p for p in passes if check_symbol_mapping(p)] - asm_shell_pre, asm_shell_post = minifier.minify_shell(asm_shell, '--minify-whitespace' in passes).split('EMSCRIPTEN_FUNCS();') + asm_shell_pre, asm_shell_post = minifier.minify_shell(asm_shell, '--minify-whitespace' in passes).split( + 'EMSCRIPTEN_FUNCS();' + ) asm_shell_post = asm_shell_post.replace('});', '})') pre += asm_shell_pre + '\n' + start_funcs_marker post = end_funcs_marker + asm_shell_post + post @@ -253,7 +262,17 @@ def check_symbol_mapping(p): lengths = [len(c) for c in chunks] if not lengths: lengths = [0] - print('chunkification: num funcs:', len(funcs), 'actual num chunks:', len(chunks), 'chunk size range:', max(lengths), '-', min(lengths), file=sys.stderr) + print( + 'chunkification: num funcs:', + len(funcs), + 'actual num chunks:', + len(chunks), + 'chunk size range:', + max(lengths), + '-', + min(lengths), + file=sys.stderr, + ) funcs = None serialized_extra_info = '' @@ -263,10 +282,12 @@ def check_symbol_mapping(p): elif extra_info: serialized_extra_info += '// EXTRA_INFO:' + json.dumps(extra_info) with ToolchainProfiler.profile_block('js_optimizer.write_chunks'): + def write_chunk(chunk, i): temp_file = temp_files.get('.jsfunc_%d.js' % i).name utils.write_file(temp_file, chunk + serialized_extra_info) return temp_file + filenames = [write_chunk(chunk, i) for i, chunk in enumerate(chunks)] with ToolchainProfiler.profile_block('run_optimizer'): @@ -314,11 +335,17 @@ def write_chunk(chunk, i): USELESS_CODE_COMMENT_BODY = 'uselessCode' brace = pre_2.find('{') + 1 has_useless_code_comment = False - if pre_2[brace:brace + len(USELESS_CODE_COMMENT_BODY)] == USELESS_CODE_COMMENT_BODY: + if pre_2[brace : brace + len(USELESS_CODE_COMMENT_BODY)] == USELESS_CODE_COMMENT_BODY: brace = pre_2.find('{', brace) + 1 has_useless_code_comment = True - pre = coutput[:start] + '(' + (USELESS_CODE_COMMENT if has_useless_code_comment else '') + 'function(global,env,buffer) {\n' + pre_2[brace:] - post = post_1 + end_asm + coutput[end + 1:] + pre = ( + coutput[:start] + + '(' + + (USELESS_CODE_COMMENT if has_useless_code_comment else '') + + 'function(global,env,buffer) {\n' + + pre_2[brace:] + ) + post = post_1 + end_asm + coutput[end + 1 :] filename += '.jo.js' temp_files.note(filename) diff --git a/tools/line_endings.py b/tools/line_endings.py index 9d883d8a52264..5d0ffb857e71c 100755 --- a/tools/line_endings.py +++ b/tools/line_endings.py @@ -16,7 +16,7 @@ def convert_line_endings(text, from_eol, to_eol): def convert_line_endings_in_file(filename, from_eol, to_eol): if from_eol == to_eol: - return # No conversion needed + return # No conversion needed with open(filename, 'rb') as f: text = f.read() @@ -45,9 +45,9 @@ def check_line_endings(filename, expect_only=None, print_errors=True, print_info if index != -1: if print_errors: print("File '" + filename + "' contains BAD line endings of form \\r\\r\\n!", file=sys.stderr) - bad_line = data[index - 50:index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') + bad_line = data[index - 50 : index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') print("Content around the location: '" + bad_line.decode('utf-8') + "'", file=sys.stderr) - return 1 # Bad line endings in file, return a non-zero process exit code. + return 1 # Bad line endings in file, return a non-zero process exit code. has_dos_line_endings = False has_unix_line_endings = False @@ -58,7 +58,7 @@ def check_line_endings(filename, expect_only=None, print_errors=True, print_info index = data.find(b'\r\n') if index != -1: - dos_line_ending_example = data[index - 50:index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') + dos_line_ending_example = data[index - 50 : index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') dos_line_ending_count = data.count(b'\r\n') # Replace all DOS line endings with some other character, and continue testing what's left. data = data.replace(b'\r\n', b'A') @@ -66,22 +66,33 @@ def check_line_endings(filename, expect_only=None, print_errors=True, print_info index = data.find(b'\r\n') if index != -1: - unix_line_ending_example = data[index - 50:index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') + unix_line_ending_example = data[index - 50 : index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') unix_line_ending_count = data.count(b'\n') has_unix_line_endings = True index = data.find(b'\r') if index != -1: - old_macos_line_ending_example = data[index - 50:index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') + old_macos_line_ending_example = data[index - 50 : index + 50].replace(b'\r', b'\\r').replace(b'\n', b'\\n') if print_errors: print('File \'' + filename + '\' contains OLD macOS line endings "\\r"', file=sys.stderr) - print("Content around an OLD macOS line ending location: '" + old_macos_line_ending_example + "'", file=sys.stderr) + print( + "Content around an OLD macOS line ending location: '" + old_macos_line_ending_example + "'", file=sys.stderr + ) # We don't want to use the old macOS (9.x) line endings anywhere. return 1 if has_dos_line_endings and has_unix_line_endings: if print_errors: - print('File \'' + filename + '\' contains both DOS "\\r\\n" and UNIX "\\n" line endings! (' + str(dos_line_ending_count) + ' DOS line endings, ' + str(unix_line_ending_count) + ' UNIX line endings)', file=sys.stderr) + print( + 'File \'' + + filename + + '\' contains both DOS "\\r\\n" and UNIX "\\n" line endings! (' + + str(dos_line_ending_count) + + ' DOS line endings, ' + + str(unix_line_ending_count) + + ' UNIX line endings)', + file=sys.stderr, + ) print("Content around a DOS line ending location: '" + dos_line_ending_example + "'", file=sys.stderr) print("Content around an UNIX line ending location: '" + unix_line_ending_example + "'", file=sys.stderr) # Mixed line endings @@ -95,15 +106,29 @@ def check_line_endings(filename, expect_only=None, print_errors=True, print_info if expect_only == '\n' and has_dos_line_endings: if print_errors: - print('File \'' + filename + '\' contains DOS "\\r\\n" line endings! (' + str(dos_line_ending_count) + ' DOS line endings), but expected only UNIX line endings!', file=sys.stderr) + print( + 'File \'' + + filename + + '\' contains DOS "\\r\\n" line endings! (' + + str(dos_line_ending_count) + + ' DOS line endings), but expected only UNIX line endings!', + file=sys.stderr, + ) print("Content around a DOS line ending location: '" + dos_line_ending_example + "'", file=sys.stderr) - return 1 # DOS line endings, but expected UNIX + return 1 # DOS line endings, but expected UNIX if expect_only == '\r\n' and has_unix_line_endings: if print_errors: - print('File \'' + filename + '\' contains UNIX "\\n" line endings! (' + str(unix_line_ending_count) + ' UNIX line endings), but expected only DOS line endings!', file=sys.stderr) + print( + 'File \'' + + filename + + '\' contains UNIX "\\n" line endings! (' + + str(unix_line_ending_count) + + ' UNIX line endings), but expected only DOS line endings!', + file=sys.stderr, + ) print("Content around a UNIX line ending location: '" + unix_line_ending_example + "'", file=sys.stderr) - return 1 # UNIX line endings, but expected DOS + return 1 # UNIX line endings, but expected DOS return 0 diff --git a/tools/maint/add_license.py b/tools/maint/add_license.py index afd27ba03afd6..149a8d528599f 100755 --- a/tools/maint/add_license.py +++ b/tools/maint/add_license.py @@ -46,13 +46,13 @@ ''' exclude_filenames = [ - 'system/include/', - 'system/lib/libc/musl/', - 'system/lib/html5/dom_pk_codes.c', - 'system/lib/dlmalloc.c', - 'third_party/', - 'test/optimizer/', - 'site/source/_themes/', + 'system/include/', + 'system/lib/libc/musl/', + 'system/lib/html5/dom_pk_codes.c', + 'system/lib/dlmalloc.c', + 'third_party/', + 'test/optimizer/', + 'site/source/_themes/', ] exclude_contents = ['Copyright', 'LICENSE.TXT', 'PUBLIC DOMAIN'] @@ -86,13 +86,13 @@ def process_file(filename): if not contents.startswith('\n'): f.write('\n') elif ext in ('.cpp', '.js'): - if contents.startswith('/*\n'): - contents = contents[3:] - f.write(c_license_base % year) - else: - f.write(cpp_license % year) - if not contents.startswith('\n'): - f.write('\n') + if contents.startswith('/*\n'): + contents = contents[3:] + f.write(c_license_base % year) + else: + f.write(cpp_license % year) + if not contents.startswith('\n'): + f.write('\n') else: assert False f.write(contents) diff --git a/tools/maint/create_entry_points.py b/tools/maint/create_entry_points.py index 03ad860e3983b..9547e27c3feb6 100755 --- a/tools/maint/create_entry_points.py +++ b/tools/maint/create_entry_points.py @@ -80,7 +80,9 @@ def generate_entry_points(cmd, path): if entry_point in entry_remap: sh_data = sh_data.replace('$0', '$(dirname $0)/' + entry_remap[entry_point]) bat_data = bat_data.replace('%~n0', entry_remap[entry_point].replace('/', '\\')) - ps1_data = ps1_data.replace(r"$MyInvocation.MyCommand.Path -replace '\.ps1$', '.py'", fr'"$PSScriptRoot/{entry_remap[entry_point]}.py"') + ps1_data = ps1_data.replace( + r"$MyInvocation.MyCommand.Path -replace '\.ps1$', '.py'", rf'"$PSScriptRoot/{entry_remap[entry_point]}.py"' + ) out_sh_file = os.path.join(__rootdir__, entry_point) with open(out_sh_file, 'w') as f: diff --git a/tools/maint/create_release.py b/tools/maint/create_release.py index a1ddc05958253..af144040857c0 100755 --- a/tools/maint/create_release.py +++ b/tools/maint/create_release.py @@ -25,8 +25,7 @@ def main(argv): utils.set_version_globals() - release_version = [utils.EMSCRIPTEN_VERSION_MAJOR, utils.EMSCRIPTEN_VERSION_MINOR, - utils.EMSCRIPTEN_VERSION_TINY] + release_version = [utils.EMSCRIPTEN_VERSION_MAJOR, utils.EMSCRIPTEN_VERSION_MINOR, utils.EMSCRIPTEN_VERSION_TINY] new_dev_version = list(release_version) new_dev_version[2] += 1 @@ -60,10 +59,10 @@ def main(argv): branch_name = 'version_' + release_version - if is_github_runner: # For GitHub Actions workflows + if is_github_runner: # For GitHub Actions workflows with open(os.environ['GITHUB_ENV'], 'a') as f: f.write(f'RELEASE_VERSION={release_version}') - else: # Local use + else: # Local use # Create a new git branch subprocess.check_call(['git', 'checkout', '-b', branch_name, 'upstream/main'], cwd=root_dir) diff --git a/tools/maint/gen_sig_info.py b/tools/maint/gen_sig_info.py index 742fbf2db2585..2d937a9740f83 100755 --- a/tools/maint/gen_sig_info.py +++ b/tools/maint/gen_sig_info.py @@ -174,12 +174,25 @@ def ignore_symbol(s, cxx): return True if s.startswith('gl') and any(s.endswith(x) for x in ('NV', 'EXT', 'WEBGL', 'ARB', 'ANGLE')): return True - if s in {'__stack_base', '__memory_base', '__table_base', '__global_base', '__heap_base', - '__stack_pointer', '__stack_high', '__stack_low', '_load_secondary_module', - '__asyncify_state', '__asyncify_data', - # legacy aliases, not callable from native code. - 'stackSave', 'stackRestore', 'stackAlloc', 'getTempRet0', 'setTempRet0', - }: + if s in { + '__stack_base', + '__memory_base', + '__table_base', + '__global_base', + '__heap_base', + '__stack_pointer', + '__stack_high', + '__stack_low', + '_load_secondary_module', + '__asyncify_state', + '__asyncify_data', + # legacy aliases, not callable from native code. + 'stackSave', + 'stackRestore', + 'stackAlloc', + 'getTempRet0', + 'setTempRet0', + }: return True return cxx and s == '__asctime_r' or s.startswith('__cxa_find_matching_catch') @@ -222,19 +235,15 @@ def functype_to_str(t, t64): def write_sig_library(filename, sig_info): - lines = [ - '/* Auto-generated by tools/gen_sig_info.py. DO NOT EDIT. */', - '', - 'sigs = {' - ] + lines = ['/* Auto-generated by tools/gen_sig_info.py. DO NOT EDIT. */', '', 'sigs = {'] for s, sig in sorted(sig_info.items()): lines.append(f" {s}__sig: '{sig}',") lines += [ - '}', - '', - '// We have to merge with `allowMissing` since this file contains signatures', - '// for functions that might not exist in all build configurations.', - 'addToLibrary(sigs, {allowMissing: true});' + '}', + '', + '// We have to merge with `allowMissing` since this file contains signatures', + '// for functions that might not exist in all build configurations.', + 'addToLibrary(sigs, {allowMissing: true});', ] utils.write_file(filename, '\n'.join(lines) + '\n') @@ -308,15 +317,18 @@ def extract_sig_info(sig_info, extra_settings=None, extra_cflags=None, cxx=False 'src/library_pthread.js', 'src/library_trace.js', ], - 'SUPPORT_LONGJMP': 'emscripten' + 'SUPPORT_LONGJMP': 'emscripten', } if extra_settings: settings.update(extra_settings) with tempfiles.get_file('.json') as settings_json: utils.write_file(settings_json, json.dumps(settings)) - output = shared.run_js_tool(utils.path_from_root('src/compiler.mjs'), - ['--symbols-only', settings_json], - stdout=subprocess.PIPE, cwd=utils.path_from_root()) + output = shared.run_js_tool( + utils.path_from_root('src/compiler.mjs'), + ['--symbols-only', settings_json], + stdout=subprocess.PIPE, + cwd=utils.path_from_root(), + ) symbols = json.loads(output)['deps'].keys() symbols = [s for s in symbols if not ignore_symbol(s, cxx)] if cxx: @@ -335,18 +347,26 @@ def extract_sig_info(sig_info, extra_settings=None, extra_cflags=None, cxx=False # The second build build allows us to determine which args/returns are pointers # or `size_t` types. These get marked as `p` in the `__sig`. obj_file = 'out.o' - cmd = [compiler, c_file, '-c', '-pthread', - '--tracing', - '-Wno-deprecated-declarations', - '-I' + utils.path_from_root('system/lib/libc'), - '-I' + utils.path_from_root('system/lib/wasmfs'), - '-o', obj_file] + cmd = [ + compiler, + c_file, + '-c', + '-pthread', + '--tracing', + '-Wno-deprecated-declarations', + '-I' + utils.path_from_root('system/lib/libc'), + '-I' + utils.path_from_root('system/lib/wasmfs'), + '-o', + obj_file, + ] if not cxx: - cmd += ['-I' + utils.path_from_root('system/lib/pthread'), - '-I' + utils.path_from_root('system/lib/libc/musl/src/include'), - '-I' + utils.path_from_root('system/lib/libc/musl/src/internal'), - '-I' + utils.path_from_root('system/lib/gl'), - '-I' + utils.path_from_root('system/lib/libcxxabi/include')] + cmd += [ + '-I' + utils.path_from_root('system/lib/pthread'), + '-I' + utils.path_from_root('system/lib/libc/musl/src/include'), + '-I' + utils.path_from_root('system/lib/libc/musl/src/internal'), + '-I' + utils.path_from_root('system/lib/gl'), + '-I' + utils.path_from_root('system/lib/libcxxabi/include'), + ] if extra_cflags: cmd += extra_cflags shared.check_call(cmd) @@ -371,26 +391,49 @@ def extract_sig_info(sig_info, extra_settings=None, extra_cflags=None, cxx=False def main(args): parser = argparse.ArgumentParser() parser.add_argument('-o', '--output', default='src/library_sigs.js') - parser.add_argument('-r', '--remove', action='store_true', help='remove from JS library files any `__sig` entries that are part of the auto-generated file') - parser.add_argument('-u', '--update', action='store_true', help='update with JS library files any `__sig` entries that are part of the auto-generated file') + parser.add_argument( + '-r', + '--remove', + action='store_true', + help='remove from JS library files any `__sig` entries that are part of the auto-generated file', + ) + parser.add_argument( + '-u', + '--update', + action='store_true', + help='update with JS library files any `__sig` entries that are part of the auto-generated file', + ) args = parser.parse_args() print('generating signatures ...') sig_info = {} - extract_sig_info(sig_info, {'WASMFS': 1, - 'JS_LIBRARIES': [], - 'USE_SDL': 0, - 'MAX_WEBGL_VERSION': 0, - 'BUILD_AS_WORKER': 1, - 'LINK_AS_CXX': 1, - 'AUTO_JS_LIBRARIES': 0}, cxx=True) + extract_sig_info( + sig_info, + { + 'WASMFS': 1, + 'JS_LIBRARIES': [], + 'USE_SDL': 0, + 'MAX_WEBGL_VERSION': 0, + 'BUILD_AS_WORKER': 1, + 'LINK_AS_CXX': 1, + 'AUTO_JS_LIBRARIES': 0, + }, + cxx=True, + ) extract_sig_info(sig_info, {'WASM_WORKERS': 1, 'JS_LIBRARIES': ['src/library_wasm_worker.js']}) extract_sig_info(sig_info, {'USE_GLFW': 3}, ['-DGLFW3']) - extract_sig_info(sig_info, {'JS_LIBRARIES': ['src/embind/embind.js', 'src/embind/emval.js'], - 'USE_SDL': 0, - 'MAX_WEBGL_VERSION': 0, - 'AUTO_JS_LIBRARIES': 0, - 'ASYNCIFY': 1}, cxx=True, extra_cflags=['-std=c++20']) + extract_sig_info( + sig_info, + { + 'JS_LIBRARIES': ['src/embind/embind.js', 'src/embind/emval.js'], + 'USE_SDL': 0, + 'MAX_WEBGL_VERSION': 0, + 'AUTO_JS_LIBRARIES': 0, + 'ASYNCIFY': 1, + }, + cxx=True, + extra_cflags=['-std=c++20'], + ) extract_sig_info(sig_info, {'LEGACY_GL_EMULATION': 1}, ['-DGLES']) extract_sig_info(sig_info, {'USE_GLFW': 2, 'FULL_ES3': 1, 'MAX_WEBGL_VERSION': 2}) extract_sig_info(sig_info, {'STANDALONE_WASM': 1}) diff --git a/tools/maint/gen_struct_info.py b/tools/maint/gen_struct_info.py index 70754f35d1b9d..1c0179fd6a7a9 100755 --- a/tools/maint/gen_struct_info.py +++ b/tools/maint/gen_struct_info.py @@ -73,32 +73,32 @@ from tools import system_libs from tools import utils -QUIET = (__name__ != '__main__') +QUIET = __name__ != '__main__' DEBUG = False CFLAGS = [ - # Avoid parsing problems due to gcc specific syntax. - '-D_GNU_SOURCE', + # Avoid parsing problems due to gcc specific syntax. + '-D_GNU_SOURCE', ] INTERNAL_CFLAGS = [ - '-I' + utils.path_from_root('system/lib/libc/musl/src/internal'), - '-I' + utils.path_from_root('system/lib/libc/musl/src/include'), - '-I' + utils.path_from_root('system/lib/pthread/'), + '-I' + utils.path_from_root('system/lib/libc/musl/src/internal'), + '-I' + utils.path_from_root('system/lib/libc/musl/src/include'), + '-I' + utils.path_from_root('system/lib/pthread/'), ] CXXFLAGS = [ - '-I' + utils.path_from_root('system/lib/libcxxabi/src'), - '-D__EMSCRIPTEN_EXCEPTIONS__', - '-I' + utils.path_from_root('system/lib/wasmfs/'), - '-std=c++17', + '-I' + utils.path_from_root('system/lib/libcxxabi/src'), + '-D__EMSCRIPTEN_EXCEPTIONS__', + '-I' + utils.path_from_root('system/lib/wasmfs/'), + '-std=c++17', ] DEFAULT_JSON_FILES = [ - utils.path_from_root('src/struct_info.json'), - utils.path_from_root('src/struct_info_internal.json'), - utils.path_from_root('src/struct_info_cxx.json'), - utils.path_from_root('src/struct_info_webgpu.json'), + utils.path_from_root('src/struct_info.json'), + utils.path_from_root('src/struct_info_internal.json'), + utils.path_from_root('src/struct_info_cxx.json'), + utils.path_from_root('src/struct_info_webgpu.json'), ] @@ -250,16 +250,25 @@ def generate_cmd(js_file_path, src_file_path, cflags): node_flags = building.get_emcc_node_flags(shared.check_node_version()) # -O1+ produces calls to iprintf, which libcompiler_rt doesn't support - cmd = [compiler] + cflags + ['-o', js_file_path, src_file_path, - '-O0', - '-Werror', - '-Wno-format', - '-nolibc', - '-sBOOTSTRAPPING_STRUCT_INFO', - '-sINCOMING_MODULE_JS_API=', - '-sSTRICT', - '-sSUPPORT_LONGJMP=0', - '-sASSERTIONS=0'] + node_flags + cmd = ( + [compiler] + + cflags + + [ + '-o', + js_file_path, + src_file_path, + '-O0', + '-Werror', + '-Wno-format', + '-nolibc', + '-sBOOTSTRAPPING_STRUCT_INFO', + '-sINCOMING_MODULE_JS_API=', + '-sSTRICT', + '-sSUPPORT_LONGJMP=0', + '-sASSERTIONS=0', + ] + + node_flags + ) # Default behavior for emcc is to warn for binaryen version check mismatches # so we should try to match that behavior. @@ -350,7 +359,9 @@ def parse_json(path): header = {'name': item['file'], 'structs': {}, 'defines': {}} for name, data in item.get('structs', {}).items(): if name in header['structs']: - show('WARN: Description of struct "' + name + '" in file "' + item['file'] + '" replaces an existing description!') + show( + 'WARN: Description of struct "' + name + '" in file "' + item['file'] + '" replaces an existing description!' + ) header['structs'][name] = data @@ -360,7 +371,13 @@ def parse_json(path): part = ['i', part] if part[1] in header['defines']: - show('WARN: Description of define "' + part[1] + '" in file "' + item['file'] + '" replaces an existing description!') + show( + 'WARN: Description of define "' + + part[1] + + '" in file "' + + item['file'] + + '" replaces an existing description!' + ) header['defines'][part[1]] = part[0] @@ -379,21 +396,32 @@ def main(args): global QUIET parser = argparse.ArgumentParser(description='Generate JSON infos for structs.') - parser.add_argument('json', nargs='*', - help='JSON file with a list of structs and their fields (defaults to src/struct_info.json)', - default=DEFAULT_JSON_FILES) - parser.add_argument('-q', dest='quiet', action='store_true', default=False, - help='Don\'t output anything besides error messages.') - parser.add_argument('-o', dest='output', metavar='path', default=None, - help='Path to the JSON file that will be written. If omitted, the default location under `src` will be used.') - parser.add_argument('-I', dest='includes', metavar='dir', action='append', default=[], - help='Add directory to include search path') - parser.add_argument('-D', dest='defines', metavar='define', action='append', default=[], - help='Pass a define to the preprocessor') - parser.add_argument('-U', dest='undefines', metavar='undefine', action='append', default=[], - help='Pass an undefine to the preprocessor') - parser.add_argument('--wasm64', action='store_true', - help='use wasm64 architecture') + parser.add_argument( + 'json', + nargs='*', + help='JSON file with a list of structs and their fields (defaults to src/struct_info.json)', + default=DEFAULT_JSON_FILES, + ) + parser.add_argument( + '-q', dest='quiet', action='store_true', default=False, help='Don\'t output anything besides error messages.' + ) + parser.add_argument( + '-o', + dest='output', + metavar='path', + default=None, + help='Path to the JSON file that will be written. If omitted, the default location under `src` will be used.', + ) + parser.add_argument( + '-I', dest='includes', metavar='dir', action='append', default=[], help='Add directory to include search path' + ) + parser.add_argument( + '-D', dest='defines', metavar='define', action='append', default=[], help='Pass a define to the preprocessor' + ) + parser.add_argument( + '-U', dest='undefines', metavar='undefine', action='append', default=[], help='Pass an undefine to the preprocessor' + ) + parser.add_argument('--wasm64', action='store_true', help='use wasm64 architecture') args = parser.parse_args(args) QUIET = args.quiet diff --git a/tools/maint/simde_update.py b/tools/maint/simde_update.py index f031c00537f48..5215fd51759c0 100755 --- a/tools/maint/simde_update.py +++ b/tools/maint/simde_update.py @@ -44,8 +44,8 @@ def main(): try: neon_h_buf = subprocess.check_output( - [path.join(simde_dir, "amalgamate.py"), path.join(simde_dir, "simde", "arm", "neon.h")], - text=True) + [path.join(simde_dir, "amalgamate.py"), path.join(simde_dir, "simde", "arm", "neon.h")], text=True + ) except subprocess.CalledProcessError as e: print("amalgamate.py returned error: " + str(e)) return 1 @@ -67,7 +67,9 @@ def main(): try: insert_location = neon_h_buf.index(line_to_prefix) except ValueError: - print(f"Error looking for place to insert {line_to_insert[:-1]!r}. Please update 'line_to_prefix' in simde_update.py.") + print( + f"Error looking for place to insert {line_to_insert[:-1]!r}. Please update 'line_to_prefix' in simde_update.py." + ) return 1 neon_h_buf = neon_h_buf[:insert_location] + line_to_insert + neon_h_buf[insert_location:] diff --git a/tools/maybe_wasm2js.py b/tools/maybe_wasm2js.py index 74fb6dd84cb14..1abb9e81643b8 100755 --- a/tools/maybe_wasm2js.py +++ b/tools/maybe_wasm2js.py @@ -45,8 +45,7 @@ cmd += opts js = shared.run_process(cmd, stdout=subprocess.PIPE).stdout # assign the instantiate function to where it will be used -js = shared.do_replace(js, 'function instantiate(info) {', - "Module['__wasm2jsInstantiate__'] = function(info) {") +js = shared.do_replace(js, 'function instantiate(info) {', "Module['__wasm2jsInstantiate__'] = function(info) {") # create the combined js to run in wasm2js mode print('var Module = { doWasm2JS: true };\n') diff --git a/tools/minimal_runtime_shell.py b/tools/minimal_runtime_shell.py index f53fa345f27c8..b872f65465898 100644 --- a/tools/minimal_runtime_shell.py +++ b/tools/minimal_runtime_shell.py @@ -29,7 +29,12 @@ def generate_minimal_runtime_load_statement(target_basename): # Expand {{{ DOWNLOAD_WASM }}} block from here (if we added #define support, this could be done in # the template directly) if settings.MINIMAL_RUNTIME_STREAMING_WASM_COMPILATION: - if settings.MIN_SAFARI_VERSION != feature_matrix.UNSUPPORTED or settings.ENVIRONMENT_MAY_BE_NODE or settings.MIN_FIREFOX_VERSION < 58 or settings.MIN_CHROME_VERSION < 61: + if ( + settings.MIN_SAFARI_VERSION != feature_matrix.UNSUPPORTED + or settings.ENVIRONMENT_MAY_BE_NODE + or settings.MIN_FIREFOX_VERSION < 58 + or settings.MIN_CHROME_VERSION < 61 + ): # Firefox 52 added Wasm support, but only Firefox 58 added compileStreaming. # Chrome 57 added Wasm support, but only Chrome 61 added compileStreaming. # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/compileStreaming @@ -41,7 +46,12 @@ def generate_minimal_runtime_load_statement(target_basename): elif settings.MINIMAL_RUNTIME_STREAMING_WASM_INSTANTIATION: # Same compatibility story as above for # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming - if settings.MIN_SAFARI_VERSION != feature_matrix.UNSUPPORTED or settings.ENVIRONMENT_MAY_BE_NODE or settings.MIN_FIREFOX_VERSION < 58 or settings.MIN_CHROME_VERSION < 61: + if ( + settings.MIN_SAFARI_VERSION != feature_matrix.UNSUPPORTED + or settings.ENVIRONMENT_MAY_BE_NODE + or settings.MIN_FIREFOX_VERSION < 58 + or settings.MIN_CHROME_VERSION < 61 + ): download_wasm = f"!WebAssembly.instantiateStreaming && binary('{target_basename}.wasm')" else: # WebAssembly.instantiateStreaming() is unconditionally supported, so we do not download wasm @@ -65,12 +75,17 @@ def generate_minimal_runtime_load_statement(target_basename): # Download wasm_worker file if settings.WASM_WORKERS: if settings.MODULARIZE: - if settings.WASM_WORKERS == 1: # '$wb': Wasm Worker Blob - modularize_imports += ["$wb: URL.createObjectURL(new Blob([r[%d]], { type: 'application/javascript' }))" % len(files_to_load)] + if settings.WASM_WORKERS == 1: # '$wb': Wasm Worker Blob + modularize_imports += [ + "$wb: URL.createObjectURL(new Blob([r[%d]], { type: 'application/javascript' }))" % len(files_to_load) + ] modularize_imports += ['js: js'] else: if settings.WASM_WORKERS == 1: - then_statements += ["%s.$wb = URL.createObjectURL(new Blob([r[%d]], { type: 'application/javascript' }));" % (settings.EXPORT_NAME, len(files_to_load))] + then_statements += [ + "%s.$wb = URL.createObjectURL(new Blob([r[%d]], { type: 'application/javascript' }));" + % (settings.EXPORT_NAME, len(files_to_load)) + ] if download_wasm and settings.WASM_WORKERS == 1: files_to_load += [f"binary('{target_basename}.ww.js')"] @@ -87,23 +102,29 @@ def generate_minimal_runtime_load_statement(target_basename): if settings.MODULARIZE: modularize_imports = ',\n '.join(modularize_imports) if settings.WASM_WORKERS: - then_statements += ['''\ + then_statements += [ + '''\ // Detour the JS code to a separate variable to avoid instantiating with 'r' array as "this" // directly to avoid strict ECMAScript/Firefox GC problems that cause a leak, see // https://bugzilla.mozilla.org/show_bug.cgi?id=1540101 var js = URL.createObjectURL(new Blob([r[0]], { type: \'application/javascript\' })); script(js).then((c) => c({ %s - }));''' % modularize_imports] + }));''' + % modularize_imports + ] else: - then_statements += ['''\ + then_statements += [ + '''\ // Detour the JS code to a separate variable to avoid instantiating with 'r' array as "this" // directly to avoid strict ECMAScript/Firefox GC problems that cause a leak, see // https://bugzilla.mozilla.org/show_bug.cgi?id=1540101 var js = r[0]; js({ %s - });''' % modularize_imports] + });''' + % modularize_imports + ] binary_xhr = ' var binary = (url) => fetch(url).then((rsp) => rsp.arrayBuffer());' @@ -166,8 +187,10 @@ def generate_minimal_runtime_load_statement(target_basename): files_to_load[0] = f"binary('{target_basename}.js')" if not settings.MODULARIZE: - then_statements += ["var url = %sURL.createObjectURL(new Blob([r[0]], { type: 'application/javascript' }));" % save_js, - script_load] + then_statements += [ + "var url = %sURL.createObjectURL(new Blob([r[0]], { type: 'application/javascript' }));" % save_js, + script_load, + ] # Add in binary() XHR loader if used: if any("binary(" in s for s in files_to_load + then_statements): @@ -190,7 +213,9 @@ def generate_minimal_runtime_html(target, options, js_target, target_basename): # No extra files needed to download in a SINGLE_FILE build. shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', '') else: - shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', generate_minimal_runtime_load_statement(target_basename)) + shell = shell.replace( + '{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', generate_minimal_runtime_load_statement(target_basename) + ) temp_files = shared.get_temp_files() with temp_files.get_file(suffix='.js') as shell_temp: @@ -198,7 +223,11 @@ def generate_minimal_runtime_html(target, options, js_target, target_basename): shell = shared.read_and_preprocess(shell_temp) if re.search(r'{{{\s*SCRIPT\s*}}}', shell): - shared.exit_with_error('--shell-file "' + options.shell_path + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.') + shared.exit_with_error( + '--shell-file "' + + options.shell_path + + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.' + ) shell = shell.replace('{{{ TARGET_BASENAME }}}', target_basename) shell = shell.replace('{{{ EXPORT_NAME }}}', settings.EXPORT_NAME) diff --git a/tools/ports/__init__.py b/tools/ports/__init__.py index c2d47ba6e96e2..3bedc29d12fd9 100644 --- a/tools/ports/__init__.py +++ b/tools/ports/__init__.py @@ -142,8 +142,7 @@ def maybe_copy(src, dest): class Ports: - """emscripten-ports library management (https://github.com/emscripten-ports). - """ + """emscripten-ports library management (https://github.com/emscripten-ports).""" @staticmethod def get_include_dir(*parts): @@ -175,7 +174,9 @@ def install_headers(src_dir, pattern='*.h', target=None): maybe_copy(f, os.path.join(dest, os.path.basename(f))) @staticmethod - def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags=[], exclude_files=[], exclude_dirs=[], srcs=[]): # noqa + def build_port( + src_dir, output_path, port_name, includes=[], flags=[], cxxflags=[], exclude_files=[], exclude_dirs=[], srcs=[] + ): # noqa build_dir = os.path.join(Ports.get_build_dir(), port_name) if srcs: srcs = [os.path.join(src_dir, s) for s in srcs] @@ -242,7 +243,7 @@ def fetch_project(name, url, sha512hash=None): # To compute the sha512 hash, run `curl URL | sha512sum`. fullname = Ports.get_dir(name) - if name not in Ports.name_cache: # only mention each port once in log + if name not in Ports.name_cache: # only mention each port once in log logger.debug(f'including port: {name}') logger.debug(f' (at {fullname})') Ports.name_cache.add(name) @@ -270,7 +271,9 @@ def fetch_project(name, url, sha512hash=None): if not port: utils.exit_with_error('%s is not a known port' % name) if not hasattr(port, 'SUBDIR'): - utils.exit_with_error(f'port {name} lacks .SUBDIR attribute, which we need in order to override it locally, please update it') + utils.exit_with_error( + f'port {name} lacks .SUBDIR attribute, which we need in order to override it locally, please update it' + ) subdir = port.SUBDIR target = os.path.join(fullname, subdir) @@ -313,8 +316,9 @@ def retrieve(): if sha512hash: actual_hash = hashlib.sha512(data).hexdigest() if actual_hash != sha512hash: - utils.exit_with_error(f'Unexpected hash: {actual_hash}\n' - 'If you are updating the port, please update the hash.') + utils.exit_with_error( + f'Unexpected hash: {actual_hash}\n' 'If you are updating the port, please update the hash.' + ) utils.write_binary(fullpath, data) marker = os.path.join(fullname, '.emscripten_url') @@ -368,6 +372,7 @@ def write_file(filename, contents): class OrderedSet: """Partial implementation of OrderedSet. Just enough for what we need here.""" + def __init__(self, items): self.dict = {} for i in items: @@ -494,15 +499,17 @@ def split_port_options(arg): # Ignore ':' in first or second char of string since we could be dealing with a windows drive separator pos = arg.find(':', 2) if pos != -1: - return arg[:pos], arg[pos + 1:] + return arg[:pos], arg[pos + 1 :] else: return arg, None def handle_use_port_arg(settings, arg, error_handler=None): if not error_handler: + def error_handler(message): handle_use_port_error(arg, message) + name, options = split_port_options(arg) if name.endswith('.py'): port_file_path = name @@ -558,7 +565,7 @@ def get_libs(settings): return ret -def add_cflags(args, settings): # noqa: U100 +def add_cflags(args, settings): # noqa: U100 """Called during compile phase add any compiler flags (e.g -Ifoo) needed by the selected ports. Can also add/change settings. diff --git a/tools/ports/boost_headers.py b/tools/ports/boost_headers.py index 738684562b584..89d2046d0eadf 100644 --- a/tools/ports/boost_headers.py +++ b/tools/ports/boost_headers.py @@ -14,9 +14,11 @@ def needed(settings): def get(ports, settings, shared): - ports.fetch_project('boost_headers', - f'https://github.com/emscripten-ports/boost/releases/download/boost-{TAG}/boost-headers-{TAG}.zip', - sha512hash=HASH) + ports.fetch_project( + 'boost_headers', + f'https://github.com/emscripten-ports/boost/releases/download/boost-{TAG}/boost-headers-{TAG}.zip', + sha512hash=HASH, + ) def create(final): # includes diff --git a/tools/ports/bullet.py b/tools/ports/bullet.py index a53028553d6e9..a3db7495c4cab 100644 --- a/tools/ports/bullet.py +++ b/tools/ports/bullet.py @@ -37,10 +37,7 @@ def create(final): for dir in dirs: includes.append(os.path.join(base, dir)) - flags = [ - '-Wno-single-bit-bitfield-constant-conversion', - '-std=gnu++14' - ] + flags = ['-Wno-single-bit-bitfield-constant-conversion', '-std=gnu++14'] ports.build_port(src_path, final, 'bullet', includes=includes, flags=flags, exclude_dirs=['MiniCL']) diff --git a/tools/ports/bzip2.py b/tools/ports/bzip2.py index 076e24cf8ac60..7ed340a45d618 100644 --- a/tools/ports/bzip2.py +++ b/tools/ports/bzip2.py @@ -20,8 +20,13 @@ def create(final): # build srcs = [ - 'blocksort.c', 'compress.c', 'decompress.c', 'huffman.c', - 'randtable.c', 'bzlib.c', 'crctable.c', + 'blocksort.c', + 'compress.c', + 'decompress.c', + 'huffman.c', + 'randtable.c', + 'bzlib.c', + 'crctable.c', ] ports.build_port(source_path, final, 'bzip2', srcs=srcs) diff --git a/tools/ports/cocos2d.py b/tools/ports/cocos2d.py index 179e14d6804bd..10bdfa4017773 100644 --- a/tools/ports/cocos2d.py +++ b/tools/ports/cocos2d.py @@ -21,7 +21,9 @@ def get(ports, settings, shared): ports.fetch_project('cocos2d', f'https://github.com/emscripten-ports/Cocos2d/archive/{TAG}.zip', sha512hash=HASH) def create(final): - diagnostics.warning('experimental', 'cocos2d: library is experimental, do not expect that it will work out of the box') + diagnostics.warning( + 'experimental', 'cocos2d: library is experimental, do not expect that it will work out of the box' + ) cocos2d_src = ports.get_dir('cocos2d') cocos2d_root = os.path.join(cocos2d_src, 'Cocos2d-' + TAG) @@ -35,7 +37,7 @@ def create(final): '-DCC_ENABLE_CHIPMUNK_INTEGRATION', '-DCC_KEYBOARD_SUPPORT', '-DGL_ES=1', - '-DNDEBUG', # '-DCOCOS2D_DEBUG=1' 1 - error/warn, 2 - verbose + '-DNDEBUG', # '-DCOCOS2D_DEBUG=1' 1 - error/warn, 2 - verbose # Cocos2d source code hasn't switched to __EMSCRIPTEN__. # See https://github.com/emscripten-ports/Cocos2d/pull/3 '-DEMSCRIPTEN', @@ -48,11 +50,7 @@ def create(final): target = os.path.join('cocos2d', os.path.relpath(dirname, cocos2d_root)) ports.install_header_dir(dirname, target=target) - ports.build_port(cocos2d_src, final, 'cocos2d', - flags=flags, - cxxflags=['-std=c++14'], - includes=includes, - srcs=srcs) + ports.build_port(cocos2d_src, final, 'cocos2d', flags=flags, cxxflags=['-std=c++14'], includes=includes, srcs=srcs) return [shared.cache.get_lib('libcocos2d.a', create, what='port')] @@ -106,34 +104,36 @@ def add_makefile(makefile): add_makefile(os.path.join(cocos2dx_root, 'platform', 'third_party', 'Makefile')) # misc sources.append(os.path.join(cocos2d_root, 'CocosDenshion', 'emscripten', 'SimpleAudioEngine.cpp')) - sources.append(os.path.join(cocos2dx_root, 'CCDeprecated.cpp')) # subset of cocos2d v2 + sources.append(os.path.join(cocos2dx_root, 'CCDeprecated.cpp')) # subset of cocos2d v2 return sources def make_includes(root): - return [os.path.join(root, 'CocosDenshion', 'include'), - os.path.join(root, 'extensions'), - os.path.join(root, 'extensions', 'AssetsManager'), - os.path.join(root, 'extensions', 'CCArmature'), - os.path.join(root, 'extensions', 'CCBReader'), - os.path.join(root, 'extensions', 'GUI', 'CCControlExtension'), - os.path.join(root, 'extensions', 'GUI', 'CCEditBox'), - os.path.join(root, 'extensions', 'GUI', 'CCScrollView'), - os.path.join(root, 'extensions', 'network'), - os.path.join(root, 'extensions', 'Components'), - os.path.join(root, 'extensions', 'LocalStorage'), - os.path.join(root, 'extensions', 'physics_nodes'), - os.path.join(root, 'extensions', 'spine'), - os.path.join(root, 'external'), - os.path.join(root, 'external', 'chipmunk', 'include', 'chipmunk'), - os.path.join(root, 'cocos2dx'), - os.path.join(root, 'cocos2dx', 'cocoa'), - os.path.join(root, 'cocos2dx', 'include'), - os.path.join(root, 'cocos2dx', 'kazmath', 'include'), - os.path.join(root, 'cocos2dx', 'platform'), - os.path.join(root, 'cocos2dx', 'platform', 'emscripten'), - os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'linux', 'libfreetype2'), - os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'common', 'etc'), - os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libtiff', 'include'), - os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libjpeg'), - os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libwebp')] + return [ + os.path.join(root, 'CocosDenshion', 'include'), + os.path.join(root, 'extensions'), + os.path.join(root, 'extensions', 'AssetsManager'), + os.path.join(root, 'extensions', 'CCArmature'), + os.path.join(root, 'extensions', 'CCBReader'), + os.path.join(root, 'extensions', 'GUI', 'CCControlExtension'), + os.path.join(root, 'extensions', 'GUI', 'CCEditBox'), + os.path.join(root, 'extensions', 'GUI', 'CCScrollView'), + os.path.join(root, 'extensions', 'network'), + os.path.join(root, 'extensions', 'Components'), + os.path.join(root, 'extensions', 'LocalStorage'), + os.path.join(root, 'extensions', 'physics_nodes'), + os.path.join(root, 'extensions', 'spine'), + os.path.join(root, 'external'), + os.path.join(root, 'external', 'chipmunk', 'include', 'chipmunk'), + os.path.join(root, 'cocos2dx'), + os.path.join(root, 'cocos2dx', 'cocoa'), + os.path.join(root, 'cocos2dx', 'include'), + os.path.join(root, 'cocos2dx', 'kazmath', 'include'), + os.path.join(root, 'cocos2dx', 'platform'), + os.path.join(root, 'cocos2dx', 'platform', 'emscripten'), + os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'linux', 'libfreetype2'), + os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'common', 'etc'), + os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libtiff', 'include'), + os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libjpeg'), + os.path.join(root, 'cocos2dx', 'platform', 'third_party', 'emscripten', 'libwebp'), + ] diff --git a/tools/ports/contrib/glfw3.py b/tools/ports/contrib/glfw3.py index 25731bac3324f..8266701039321 100644 --- a/tools/ports/contrib/glfw3.py +++ b/tools/ports/contrib/glfw3.py @@ -18,7 +18,7 @@ 'disableWarning': ['true', 'false'], 'disableJoystick': ['true', 'false'], 'disableMultiWindow': ['true', 'false'], - 'optimizationLevel': ['0', '1', '2', '3', 'g', 's', 'z'] # all -OX possibilities + 'optimizationLevel': ['0', '1', '2', '3', 'g', 's', 'z'], # all -OX possibilities } OPTIONS = { @@ -33,25 +33,29 @@ 'disableWarning': False, 'disableJoystick': False, 'disableMultiWindow': False, - 'optimizationLevel': '2' + 'optimizationLevel': '2', } port_name = 'contrib.glfw3' def get_lib_name(settings): - return (f'lib_{port_name}-O{opts["optimizationLevel"]}' + - ('-nw' if opts['disableWarning'] else '') + - ('-nj' if opts['disableJoystick'] else '') + - ('-sw' if opts['disableMultiWindow'] else '') + - '.a') + return ( + f'lib_{port_name}-O{opts["optimizationLevel"]}' + + ('-nw' if opts['disableWarning'] else '') + + ('-nj' if opts['disableJoystick'] else '') + + ('-sw' if opts['disableMultiWindow'] else '') + + '.a' + ) def get(ports, settings, shared): # get the port - ports.fetch_project(port_name, - f'https://github.com/pongasoft/emscripten-glfw/releases/download/v{TAG}/emscripten-glfw3-{TAG}.zip', - sha512hash=HASH) + ports.fetch_project( + port_name, + f'https://github.com/pongasoft/emscripten-glfw/releases/download/v{TAG}/emscripten-glfw3-{TAG}.zip', + sha512hash=HASH, + ) def create(final): root_path = os.path.join(ports.get_dir(), port_name) diff --git a/tools/ports/freetype.py b/tools/ports/freetype.py index 22bfb6aa938b5..6dc0a937c4874 100644 --- a/tools/ports/freetype.py +++ b/tools/ports/freetype.py @@ -29,52 +29,53 @@ def get(ports, settings, shared): def create(final): source_path = ports.get_dir('freetype', f'freetype-{TAG}') # Overwrite the default config file with our own. - ports.install_header_dir(os.path.join(source_path, 'include'), - target=os.path.join('freetype2')) + ports.install_header_dir(os.path.join(source_path, 'include'), target=os.path.join('freetype2')) # build - srcs = ['builds/unix/ftsystem.c', - 'src/autofit/autofit.c', - 'src/base/ftbase.c', - 'src/base/ftbbox.c', - 'src/base/ftbdf.c', - 'src/base/ftbitmap.c', - 'src/base/ftcid.c', - 'src/base/ftdebug.c', - 'src/base/ftfstype.c', - 'src/base/ftgasp.c', - 'src/base/ftglyph.c', - 'src/base/ftgxval.c', - 'src/base/ftinit.c', - 'src/base/ftmm.c', - 'src/base/ftotval.c', - 'src/base/ftpatent.c', - 'src/base/ftpfr.c', - 'src/base/ftstroke.c', - 'src/base/ftsynth.c', - 'src/base/fttype1.c', - 'src/base/ftwinfnt.c', - 'src/bdf/bdf.c', - 'src/bzip2/ftbzip2.c', - 'src/cache/ftcache.c', - 'src/cff/cff.c', - 'src/cid/type1cid.c', - 'src/gzip/ftgzip.c', - 'src/lzw/ftlzw.c', - 'src/pcf/pcf.c', - 'src/pfr/pfr.c', - 'src/psaux/psaux.c', - 'src/pshinter/pshinter.c', - 'src/psnames/psnames.c', - 'src/raster/raster.c', - 'src/sdf/sdf.c', - 'src/sfnt/sfnt.c', - 'src/smooth/smooth.c', - 'src/svg/svg.c', - 'src/truetype/truetype.c', - 'src/type1/type1.c', - 'src/type42/type42.c', - 'src/winfonts/winfnt.c'] + srcs = [ + 'builds/unix/ftsystem.c', + 'src/autofit/autofit.c', + 'src/base/ftbase.c', + 'src/base/ftbbox.c', + 'src/base/ftbdf.c', + 'src/base/ftbitmap.c', + 'src/base/ftcid.c', + 'src/base/ftdebug.c', + 'src/base/ftfstype.c', + 'src/base/ftgasp.c', + 'src/base/ftglyph.c', + 'src/base/ftgxval.c', + 'src/base/ftinit.c', + 'src/base/ftmm.c', + 'src/base/ftotval.c', + 'src/base/ftpatent.c', + 'src/base/ftpfr.c', + 'src/base/ftstroke.c', + 'src/base/ftsynth.c', + 'src/base/fttype1.c', + 'src/base/ftwinfnt.c', + 'src/bdf/bdf.c', + 'src/bzip2/ftbzip2.c', + 'src/cache/ftcache.c', + 'src/cff/cff.c', + 'src/cid/type1cid.c', + 'src/gzip/ftgzip.c', + 'src/lzw/ftlzw.c', + 'src/pcf/pcf.c', + 'src/pfr/pfr.c', + 'src/psaux/psaux.c', + 'src/pshinter/pshinter.c', + 'src/psnames/psnames.c', + 'src/raster/raster.c', + 'src/sdf/sdf.c', + 'src/sfnt/sfnt.c', + 'src/smooth/smooth.c', + 'src/svg/svg.c', + 'src/truetype/truetype.c', + 'src/type1/type1.c', + 'src/type42/type42.c', + 'src/winfonts/winfnt.c', + ] flags = [ '-DFT2_BUILD_LIBRARY', @@ -90,7 +91,7 @@ def create(final): '-I' + source_path + '/psaux', '-I' + source_path + '/psnames', '-I' + source_path + '/truetype', - '-pthread' + '-pthread', ] if settings.SUPPORT_LONGJMP == 'wasm': diff --git a/tools/ports/giflib.py b/tools/ports/giflib.py index fb870202c3f7c..280f09ae9ca15 100644 --- a/tools/ports/giflib.py +++ b/tools/ports/giflib.py @@ -12,15 +12,32 @@ def needed(settings): def get(ports, settings, shared): - ports.fetch_project('giflib', f'https://storage.googleapis.com/webassembly/emscripten-ports/giflib-{VERSION}.tar.gz', sha512hash=HASH) + ports.fetch_project( + 'giflib', f'https://storage.googleapis.com/webassembly/emscripten-ports/giflib-{VERSION}.tar.gz', sha512hash=HASH + ) def create(final): source_path = ports.get_dir('giflib', f'giflib-{VERSION}') ports.install_headers(source_path) exclude_files = [ - 'giffix.c', 'gifecho.c', 'giffilter.c', 'gifcolor.c', 'gifecho.c', 'gifinto.c', - 'gifsponge.c', 'gif2rgb.c', 'gifbg.c', 'gifbuild.c', 'gifclrmp.c', 'gifhisto.c', - 'gifbuild.c', 'gifclrmp.c', 'gifhisto.c', 'giftext.c', 'giftool.c', 'gifwedge.c', + 'giffix.c', + 'gifecho.c', + 'giffilter.c', + 'gifcolor.c', + 'gifecho.c', + 'gifinto.c', + 'gifsponge.c', + 'gif2rgb.c', + 'gifbg.c', + 'gifbuild.c', + 'gifclrmp.c', + 'gifhisto.c', + 'gifbuild.c', + 'gifclrmp.c', + 'gifhisto.c', + 'giftext.c', + 'giftool.c', + 'gifwedge.c', ] ports.build_port(source_path, final, 'giflib', exclude_files=exclude_files) diff --git a/tools/ports/harfbuzz.py b/tools/ports/harfbuzz.py index ac1cb26109993..b2621a0fa9fea 100644 --- a/tools/ports/harfbuzz.py +++ b/tools/ports/harfbuzz.py @@ -79,7 +79,11 @@ def get_lib_name(settings): def get(ports, settings, shared): - ports.fetch_project('harfbuzz', f'https://github.com/harfbuzz/harfbuzz/releases/download/{VERSION}/harfbuzz-{VERSION}.tar.xz', sha512hash=HASH) + ports.fetch_project( + 'harfbuzz', + f'https://github.com/harfbuzz/harfbuzz/releases/download/{VERSION}/harfbuzz-{VERSION}.tar.xz', + sha512hash=HASH, + ) def create(final): source_path = ports.get_dir('harfbuzz', 'harfbuzz-' + VERSION) diff --git a/tools/ports/icu.py b/tools/ports/icu.py index 530122f3e18c2..dd12a02035604 100644 --- a/tools/ports/icu.py +++ b/tools/ports/icu.py @@ -26,32 +26,34 @@ def get_lib_name(base_name, settings): def get(ports, settings, shared): - ports.fetch_project('icu', f'https://github.com/unicode-org/icu/releases/download/{TAG}/icu4c-{VERSION}-src.zip', sha512hash=HASH) + ports.fetch_project( + 'icu', f'https://github.com/unicode-org/icu/releases/download/{TAG}/icu4c-{VERSION}-src.zip', sha512hash=HASH + ) icu_source_path = None def prepare_build(): nonlocal icu_source_path - source_path = ports.get_dir('icu', 'icu') # downloaded icu4c path + source_path = ports.get_dir('icu', 'icu') # downloaded icu4c path icu_source_path = os.path.join(source_path, 'source') def build_lib(lib_output, lib_src, other_includes, build_flags): additional_build_flags = [ - # TODO: investigate why this is needed and remove - '-Wno-macro-redefined', - '-Wno-deprecated-declarations', - '-Wno-array-compare', - '-Wno-unknown-warning-option', - # usage of 'using namespace icu' is deprecated: icu v61 - '-DU_USING_ICU_NAMESPACE=0', - # make explicit inclusion of utf header: ref utf.h - '-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1', - # mark UnicodeString constructors explicit : ref unistr.h - '-DUNISTR_FROM_CHAR_EXPLICIT=explicit', - '-DUNISTR_FROM_STRING_EXPLICIT=explicit', - # generate static - '-DU_STATIC_IMPLEMENTATION', - # CXXFLAGS - '-std=c++11' + # TODO: investigate why this is needed and remove + '-Wno-macro-redefined', + '-Wno-deprecated-declarations', + '-Wno-array-compare', + '-Wno-unknown-warning-option', + # usage of 'using namespace icu' is deprecated: icu v61 + '-DU_USING_ICU_NAMESPACE=0', + # make explicit inclusion of utf header: ref utf.h + '-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1', + # mark UnicodeString constructors explicit : ref unistr.h + '-DUNISTR_FROM_CHAR_EXPLICIT=explicit', + '-DUNISTR_FROM_STRING_EXPLICIT=explicit', + # generate static + '-DU_STATIC_IMPLEMENTATION', + # CXXFLAGS + '-std=c++11', ] if settings.PTHREADS: additional_build_flags.append('-pthread') @@ -87,10 +89,12 @@ def create_libicu_io(lib_output): build_lib(lib_output, lib_src, other_includes, ['-DU_IO_IMPLEMENTATION=1']) return [ - shared.cache.get_lib(get_lib_name(libname_libicu_common, settings), create_libicu_common), # this also prepares the build - shared.cache.get_lib(get_lib_name(libname_libicu_stubdata, settings), create_libicu_stubdata), - shared.cache.get_lib(get_lib_name(libname_libicu_i18n, settings), create_libicu_i18n), - shared.cache.get_lib(get_lib_name(libname_libicu_io, settings), create_libicu_io) + shared.cache.get_lib( + get_lib_name(libname_libicu_common, settings), create_libicu_common + ), # this also prepares the build + shared.cache.get_lib(get_lib_name(libname_libicu_stubdata, settings), create_libicu_stubdata), + shared.cache.get_lib(get_lib_name(libname_libicu_i18n, settings), create_libicu_i18n), + shared.cache.get_lib(get_lib_name(libname_libicu_io, settings), create_libicu_io), ] diff --git a/tools/ports/libjpeg.py b/tools/ports/libjpeg.py index 23edf867684bc..710a64226cc7f 100644 --- a/tools/ports/libjpeg.py +++ b/tools/ports/libjpeg.py @@ -17,16 +17,27 @@ def get(ports, settings, shared): # Archive mirrored from http://www.ijg.org/files/jpegsrc.v9c.tar.gz. # We have issues where python urllib was not able to load from the www.ijg.org webserver # and was resulting in 403: Forbidden. - ports.fetch_project('libjpeg', f'https://storage.googleapis.com/webassembly/emscripten-ports/jpegsrc.v{VERSION}.tar.gz', sha512hash=HASH) + ports.fetch_project( + 'libjpeg', f'https://storage.googleapis.com/webassembly/emscripten-ports/jpegsrc.v{VERSION}.tar.gz', sha512hash=HASH + ) def create(final): source_path = ports.get_dir('libjpeg', f'jpeg-{VERSION}') ports.write_file(os.path.join(source_path, 'jconfig.h'), jconfig_h) ports.install_headers(source_path) excludes = [ - 'ansi2knr.c', 'cjpeg.c', 'ckconfig.c', 'djpeg.c', 'example.c', - 'jmemansi.c', 'jmemdos.c', 'jmemmac.c', 'jmemname.c', - 'jpegtran.c', 'rdjpgcom.c', 'wrjpgcom.c', + 'ansi2knr.c', + 'cjpeg.c', + 'ckconfig.c', + 'djpeg.c', + 'example.c', + 'jmemansi.c', + 'jmemdos.c', + 'jmemmac.c', + 'jmemname.c', + 'jpegtran.c', + 'rdjpgcom.c', + 'wrjpgcom.c', ] ports.build_port(source_path, final, 'libjpeg', exclude_files=excludes) diff --git a/tools/ports/libpng.py b/tools/ports/libpng.py index 925a43f002763..bc62d685c4c9a 100644 --- a/tools/ports/libpng.py +++ b/tools/ports/libpng.py @@ -31,7 +31,9 @@ def get_lib_name(settings): def get(ports, settings, shared): # This is an emscripten-hosted mirror of the libpng repo from Sourceforge. - ports.fetch_project('libpng', f'https://storage.googleapis.com/webassembly/emscripten-ports/libpng-{TAG}.tar.gz', sha512hash=HASH) + ports.fetch_project( + 'libpng', f'https://storage.googleapis.com/webassembly/emscripten-ports/libpng-{TAG}.tar.gz', sha512hash=HASH + ) def create(final): source_path = ports.get_dir('libpng', 'libpng-' + TAG) @@ -44,7 +46,9 @@ def create(final): if settings.SUPPORT_LONGJMP == 'wasm': flags.append('-sSUPPORT_LONGJMP=wasm') - ports.build_port(source_path, final, 'libpng', flags=flags, exclude_files=['pngtest'], exclude_dirs=['scripts', 'contrib']) + ports.build_port( + source_path, final, 'libpng', flags=flags, exclude_files=['pngtest'], exclude_dirs=['scripts', 'contrib'] + ) return [shared.cache.get_lib(get_lib_name(settings), create, what='port')] diff --git a/tools/ports/mpg123.py b/tools/ports/mpg123.py index 6b2d4f51bc2b8..b4354a8041e03 100644 --- a/tools/ports/mpg123.py +++ b/tools/ports/mpg123.py @@ -610,7 +610,7 @@ def show(): /* Define to `unsigned long' if does not define. */ /* #undef uintptr_t */ -''' # noqa: W291,E101,W191 +''' # noqa: W291,E101,W191 mpg123_h = r'''/* libmpg123: MPEG Audio Decoder library (version 1.26.2) @@ -2309,4 +2309,4 @@ def show(): #endif #endif -''' # noqa: W291,E101,W191 +''' # noqa: W291,E101,W191 diff --git a/tools/ports/regal.py b/tools/ports/regal.py index e538bd030366e..603d88af816d1 100644 --- a/tools/ports/regal.py +++ b/tools/ports/regal.py @@ -38,60 +38,62 @@ def create(final): ports.install_headers(source_path_include, target='GL') # build - srcs_regal = ['regal/RegalShaderInstance.cpp', - 'regal/RegalIff.cpp', - 'regal/RegalQuads.cpp', - 'regal/Regal.cpp', - 'regal/RegalLog.cpp', - 'regal/RegalInit.cpp', - 'regal/RegalBreak.cpp', - 'regal/RegalUtil.cpp', - 'regal/RegalEmu.cpp', - 'regal/RegalEmuInfo.cpp', - 'regal/RegalFrame.cpp', - 'regal/RegalHelper.cpp', - 'regal/RegalMarker.cpp', - 'regal/RegalTexC.cpp', - 'regal/RegalCacheShader.cpp', - 'regal/RegalCacheTexture.cpp', - 'regal/RegalConfig.cpp', - 'regal/RegalContext.cpp', - 'regal/RegalContextInfo.cpp', - 'regal/RegalDispatch.cpp', - 'regal/RegalStatistics.cpp', - 'regal/RegalLookup.cpp', - 'regal/RegalPlugin.cpp', - 'regal/RegalShader.cpp', - 'regal/RegalToken.cpp', - 'regal/RegalDispatchGlobal.cpp', - 'regal/RegalDispatcher.cpp', - 'regal/RegalDispatcherGL.cpp', - 'regal/RegalDispatcherGlobal.cpp', - 'regal/RegalDispatchEmu.cpp', - 'regal/RegalDispatchGLX.cpp', - 'regal/RegalDispatchLog.cpp', - 'regal/RegalDispatchCode.cpp', - 'regal/RegalDispatchCache.cpp', - 'regal/RegalDispatchError.cpp', - 'regal/RegalDispatchLoader.cpp', - 'regal/RegalDispatchDebug.cpp', - 'regal/RegalDispatchPpapi.cpp', - 'regal/RegalDispatchStatistics.cpp', - 'regal/RegalDispatchStaticES2.cpp', - 'regal/RegalDispatchStaticEGL.cpp', - 'regal/RegalDispatchTrace.cpp', - 'regal/RegalDispatchMissing.cpp', - 'regal/RegalPixelConversions.cpp', - 'regal/RegalHttp.cpp', - 'regal/RegalDispatchHttp.cpp', - 'regal/RegalJson.cpp', - 'regal/RegalFavicon.cpp', - 'regal/RegalMac.cpp', - 'regal/RegalSo.cpp', - 'regal/RegalFilt.cpp', - 'regal/RegalXfer.cpp', - 'regal/RegalX11.cpp', - 'regal/RegalDllMain.cpp'] + srcs_regal = [ + 'regal/RegalShaderInstance.cpp', + 'regal/RegalIff.cpp', + 'regal/RegalQuads.cpp', + 'regal/Regal.cpp', + 'regal/RegalLog.cpp', + 'regal/RegalInit.cpp', + 'regal/RegalBreak.cpp', + 'regal/RegalUtil.cpp', + 'regal/RegalEmu.cpp', + 'regal/RegalEmuInfo.cpp', + 'regal/RegalFrame.cpp', + 'regal/RegalHelper.cpp', + 'regal/RegalMarker.cpp', + 'regal/RegalTexC.cpp', + 'regal/RegalCacheShader.cpp', + 'regal/RegalCacheTexture.cpp', + 'regal/RegalConfig.cpp', + 'regal/RegalContext.cpp', + 'regal/RegalContextInfo.cpp', + 'regal/RegalDispatch.cpp', + 'regal/RegalStatistics.cpp', + 'regal/RegalLookup.cpp', + 'regal/RegalPlugin.cpp', + 'regal/RegalShader.cpp', + 'regal/RegalToken.cpp', + 'regal/RegalDispatchGlobal.cpp', + 'regal/RegalDispatcher.cpp', + 'regal/RegalDispatcherGL.cpp', + 'regal/RegalDispatcherGlobal.cpp', + 'regal/RegalDispatchEmu.cpp', + 'regal/RegalDispatchGLX.cpp', + 'regal/RegalDispatchLog.cpp', + 'regal/RegalDispatchCode.cpp', + 'regal/RegalDispatchCache.cpp', + 'regal/RegalDispatchError.cpp', + 'regal/RegalDispatchLoader.cpp', + 'regal/RegalDispatchDebug.cpp', + 'regal/RegalDispatchPpapi.cpp', + 'regal/RegalDispatchStatistics.cpp', + 'regal/RegalDispatchStaticES2.cpp', + 'regal/RegalDispatchStaticEGL.cpp', + 'regal/RegalDispatchTrace.cpp', + 'regal/RegalDispatchMissing.cpp', + 'regal/RegalPixelConversions.cpp', + 'regal/RegalHttp.cpp', + 'regal/RegalDispatchHttp.cpp', + 'regal/RegalJson.cpp', + 'regal/RegalFavicon.cpp', + 'regal/RegalMac.cpp', + 'regal/RegalSo.cpp', + 'regal/RegalFilt.cpp', + 'regal/RegalXfer.cpp', + 'regal/RegalX11.cpp', + 'regal/RegalDllMain.cpp', + ] srcs_regal = [os.path.join(source_path_src, s) for s in srcs_regal] @@ -101,7 +103,7 @@ def create(final): '-DREGAL_MISSING=0', # Set to 1 if you don't want to crash in case of missing GL implementation '-std=gnu++14', '-fno-rtti', - '-fno-exceptions', # Disable exceptions (in STL containers mostly), as they are not used at all + '-fno-exceptions', # Disable exceptions (in STL containers mostly), as they are not used at all '-O3', '-I' + source_path_regal, '-I' + source_path_lookup3, diff --git a/tools/ports/sdl2_image.py b/tools/ports/sdl2_image.py index e36344eeddc21..d32a0ab0a0639 100644 --- a/tools/ports/sdl2_image.py +++ b/tools/ports/sdl2_image.py @@ -11,23 +11,36 @@ deps = ['sdl2'] variants = { - 'sdl2_image-jpg': {'SDL2_IMAGE_FORMATS': ["jpg"]}, - 'sdl2_image-png': {'SDL2_IMAGE_FORMATS': ["png"]}, + 'sdl2_image-jpg': {'SDL2_IMAGE_FORMATS': ["jpg"]}, + 'sdl2_image-png': {'SDL2_IMAGE_FORMATS': ["png"]}, 'sdl2_image-jpg-mt': {'SDL2_IMAGE_FORMATS': ["jpg"], 'PTHREADS': 1}, 'sdl2_image-png-mt': {'SDL2_IMAGE_FORMATS': ["png"], 'PTHREADS': 1}, } -OPTIONS = { - 'formats': 'A comma separated list of formats (ex: --use-port=sdl2_image:formats=png,jpg)' +OPTIONS = {'formats': 'A comma separated list of formats (ex: --use-port=sdl2_image:formats=png,jpg)'} + +SUPPORTED_FORMATS = { + 'avif', + 'bmp', + 'gif', + 'jpg', + 'jxl', + 'lbm', + 'pcx', + 'png', + 'pnm', + 'qoi', + 'svg', + 'tga', + 'tif', + 'webp', + 'xcf', + 'xpm', + 'xv', } -SUPPORTED_FORMATS = {'avif', 'bmp', 'gif', 'jpg', 'jxl', 'lbm', 'pcx', 'png', - 'pnm', 'qoi', 'svg', 'tga', 'tif', 'webp', 'xcf', 'xpm', 'xv'} - # user options (from --use-port) -opts: Dict[str, Set] = { - 'formats': set() -} +opts: Dict[str, Set] = {'formats': set()} def needed(settings): @@ -52,7 +65,9 @@ def get_lib_name(settings): def get(ports, settings, shared): sdl_build = os.path.join(ports.get_build_dir(), 'sdl2') assert os.path.exists(sdl_build), 'You must use SDL2 to use SDL2_image' - ports.fetch_project('sdl2_image', f'https://github.com/libsdl-org/SDL_image/archive/refs/tags/{TAG}.zip', sha512hash=HASH) + ports.fetch_project( + 'sdl2_image', f'https://github.com/libsdl-org/SDL_image/archive/refs/tags/{TAG}.zip', sha512hash=HASH + ) libname = get_lib_name(settings) def create(final): diff --git a/tools/ports/sdl2_mixer.py b/tools/ports/sdl2_mixer.py index f77f906d49fdd..a47784a1d0357 100644 --- a/tools/ports/sdl2_mixer.py +++ b/tools/ports/sdl2_mixer.py @@ -70,11 +70,7 @@ def create(final): build_dir = ports.clear_project_build('sdl2_mixer') include_path = os.path.join(source_path, 'include') - includes = [ - include_path, - os.path.join(source_path, 'src'), - os.path.join(source_path, 'src', 'codecs') - ] + includes = [include_path, os.path.join(source_path, 'src'), os.path.join(source_path, 'src', 'codecs')] ports.build_port( source_path, final, diff --git a/tools/ports/sdl2_ttf.py b/tools/ports/sdl2_ttf.py index 4e0d75f0e5330..3958b1b59cf77 100644 --- a/tools/ports/sdl2_ttf.py +++ b/tools/ports/sdl2_ttf.py @@ -3,7 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -TAG = 'release-2.20.2' # Latest as of 21 February 2023 +TAG = 'release-2.20.2' # Latest as of 21 February 2023 HASH = '8a625d29bef2ab7cbfe2143136a303c0fdb066ecd802d6c725de1b73ad8b056908cb524fe58f38eaee9f105471d2af50bbcb17911d46506dbcf573db218b3685' deps = ['freetype', 'sdl2', 'harfbuzz'] diff --git a/tools/ports/vorbis.py b/tools/ports/vorbis.py index bbde0d934dd9b..3a46f8551b66f 100644 --- a/tools/ports/vorbis.py +++ b/tools/ports/vorbis.py @@ -21,9 +21,13 @@ def get(ports, settings, shared): def create(final): source_path = ports.get_dir('vorbis', 'Vorbis-' + TAG) ports.install_headers(os.path.join(source_path, 'include', 'vorbis'), target='vorbis') - ports.build_port(os.path.join(source_path, 'lib'), final, 'vorbis', - flags=['-sUSE_OGG'], - exclude_files=['psytune', 'barkmel', 'tone', 'misc']) + ports.build_port( + os.path.join(source_path, 'lib'), + final, + 'vorbis', + flags=['-sUSE_OGG'], + exclude_files=['psytune', 'barkmel', 'tone', 'misc'], + ) return [shared.cache.get_lib('libvorbis.a', create)] diff --git a/tools/response_file.py b/tools/response_file.py index 93a2b939eba0c..fd5e2530162e2 100644 --- a/tools/response_file.py +++ b/tools/response_file.py @@ -25,7 +25,7 @@ def create_response_file(args, directory, suffix='.rsp.utf-8'): response_fd, response_filename = tempfile.mkstemp(prefix='emscripten_', suffix=suffix, dir=directory, text=True) # Backslashes and other special chars need to be escaped in the response file. - escape_chars = ['\\', '\"'] + escape_chars = ['\\', '"'] # When calling llvm-ar on Linux and macOS, single quote characters ' should be escaped. if not WINDOWS: escape_chars += ['\''] @@ -60,6 +60,7 @@ def escape(arg): # Register the created .rsp file to be automatically cleaned up once this # process finishes, so that caller does not have to remember to do it. from . import shared + shared.get_temp_files().note(response_filename) return response_filename @@ -84,7 +85,12 @@ def read_response_file(response_filename): # Guess encoding based on the file suffix components = os.path.basename(response_filename).split('.') encoding_suffix = components[-1].lower() - if len(components) > 1 and (encoding_suffix.startswith('utf') or encoding_suffix.startswith('cp') or encoding_suffix.startswith('iso') or encoding_suffix in ['ascii', 'latin-1']): + if len(components) > 1 and ( + encoding_suffix.startswith('utf') + or encoding_suffix.startswith('cp') + or encoding_suffix.startswith('iso') + or encoding_suffix in ['ascii', 'latin-1'] + ): guessed_encoding = encoding_suffix else: # On windows, recent version of CMake emit rsp files containing @@ -96,9 +102,14 @@ def read_response_file(response_filename): # First try with the guessed encoding with open(response_filename, encoding=guessed_encoding) as f: args = f.read() - except (ValueError, LookupError): # UnicodeDecodeError is a subclass of ValueError, and Python raises either a ValueError or a UnicodeDecodeError on decode errors. LookupError is raised if guessed encoding is not an encoding. + except ( + ValueError, + LookupError, + ): # UnicodeDecodeError is a subclass of ValueError, and Python raises either a ValueError or a UnicodeDecodeError on decode errors. LookupError is raised if guessed encoding is not an encoding. if DEBUG: - logging.warning(f'Failed to parse response file {response_filename} with guessed encoding "{guessed_encoding}". Trying default system encoding...') + logging.warning( + f'Failed to parse response file {response_filename} with guessed encoding "{guessed_encoding}". Trying default system encoding...' + ) # If that fails, try with the Python default locale.getpreferredencoding() with open(response_filename) as f: args = f.read() diff --git a/tools/settings.py b/tools/settings.py index 146a3ba3300ac..9903b845857b4 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -14,101 +14,98 @@ # Subset of settings that take a memory size (i.e. 1Gb, 64kb etc) MEM_SIZE_SETTINGS = { - 'GLOBAL_BASE', - 'STACK_SIZE', - 'TOTAL_STACK', - 'INITIAL_HEAP', - 'INITIAL_MEMORY', - 'MEMORY_GROWTH_LINEAR_STEP', - 'MEMORY_GROWTH_GEOMETRIC_CAP', - 'GL_MAX_TEMP_BUFFER_SIZE', - 'MAXIMUM_MEMORY', - 'DEFAULT_PTHREAD_STACK_SIZE' + 'GLOBAL_BASE', + 'STACK_SIZE', + 'TOTAL_STACK', + 'INITIAL_HEAP', + 'INITIAL_MEMORY', + 'MEMORY_GROWTH_LINEAR_STEP', + 'MEMORY_GROWTH_GEOMETRIC_CAP', + 'GL_MAX_TEMP_BUFFER_SIZE', + 'MAXIMUM_MEMORY', + 'DEFAULT_PTHREAD_STACK_SIZE', } PORTS_SETTINGS = { - # All port-related settings are valid at compile time - 'USE_SDL', - 'USE_LIBPNG', - 'USE_BULLET', - 'USE_ZLIB', - 'USE_BZIP2', - 'USE_VORBIS', - 'USE_COCOS2D', - 'USE_ICU', - 'USE_MODPLUG', - 'USE_SDL_MIXER', - 'USE_SDL_IMAGE', - 'USE_SDL_TTF', - 'USE_SDL_NET', - 'USE_SDL_GFX', - 'USE_LIBJPEG', - 'USE_OGG', - 'USE_REGAL', - 'USE_BOOST_HEADERS', - 'USE_HARFBUZZ', - 'USE_MPG123', - 'USE_GIFLIB', - 'USE_FREETYPE', - 'SDL2_MIXER_FORMATS', - 'SDL2_IMAGE_FORMATS', - 'USE_SQLITE3', + # All port-related settings are valid at compile time + 'USE_SDL', + 'USE_LIBPNG', + 'USE_BULLET', + 'USE_ZLIB', + 'USE_BZIP2', + 'USE_VORBIS', + 'USE_COCOS2D', + 'USE_ICU', + 'USE_MODPLUG', + 'USE_SDL_MIXER', + 'USE_SDL_IMAGE', + 'USE_SDL_TTF', + 'USE_SDL_NET', + 'USE_SDL_GFX', + 'USE_LIBJPEG', + 'USE_OGG', + 'USE_REGAL', + 'USE_BOOST_HEADERS', + 'USE_HARFBUZZ', + 'USE_MPG123', + 'USE_GIFLIB', + 'USE_FREETYPE', + 'SDL2_MIXER_FORMATS', + 'SDL2_IMAGE_FORMATS', + 'USE_SQLITE3', } # Subset of settings that apply only when generating JS JS_ONLY_SETTINGS = { - 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', - 'INCLUDE_FULL_LIBRARY', - 'PROXY_TO_WORKER', - 'PROXY_TO_WORKER_FILENAME', - 'BUILD_AS_WORKER', - 'STRICT_JS', - 'SMALL_XHR_CHUNKS', - 'HEADLESS', - 'MODULARIZE', - 'EXPORT_ES6', - 'USE_ES6_IMPORT_META', - 'EXPORT_NAME', - 'DYNAMIC_EXECUTION', - 'PTHREAD_POOL_SIZE', - 'PTHREAD_POOL_SIZE_STRICT', - 'PTHREAD_POOL_DELAY_LOAD', - 'DEFAULT_PTHREAD_STACK_SIZE', + 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE', + 'INCLUDE_FULL_LIBRARY', + 'PROXY_TO_WORKER', + 'PROXY_TO_WORKER_FILENAME', + 'BUILD_AS_WORKER', + 'STRICT_JS', + 'SMALL_XHR_CHUNKS', + 'HEADLESS', + 'MODULARIZE', + 'EXPORT_ES6', + 'USE_ES6_IMPORT_META', + 'EXPORT_NAME', + 'DYNAMIC_EXECUTION', + 'PTHREAD_POOL_SIZE', + 'PTHREAD_POOL_SIZE_STRICT', + 'PTHREAD_POOL_DELAY_LOAD', + 'DEFAULT_PTHREAD_STACK_SIZE', } # Subset of settings that apply at compile time. # (Keep in sync with [compile] comments in settings.js) COMPILE_TIME_SETTINGS = { - 'MEMORY64', - 'INLINING_LIMIT', - 'DISABLE_EXCEPTION_CATCHING', - 'DISABLE_EXCEPTION_THROWING', - 'MAIN_MODULE', - 'SIDE_MODULE', - 'RELOCATABLE', - 'LINKABLE', - 'STRICT', - 'EMSCRIPTEN_TRACING', - 'PTHREADS', - 'USE_PTHREADS', # legacy name of PTHREADS setting - 'SHARED_MEMORY', - 'SUPPORT_LONGJMP', - 'WASM_OBJECT_FILES', - 'WASM_WORKERS', - 'BULK_MEMORY', - - # Internal settings used during compilation - 'EXCEPTION_CATCHING_ALLOWED', - 'WASM_EXCEPTIONS', - 'LTO', - 'OPT_LEVEL', - 'DEBUG_LEVEL', - - # Affects ports - 'GL_ENABLE_GET_PROC_ADDRESS', # NOTE: if SDL2 is updated to not rely on eglGetProcAddress(), this can be removed - - # This is legacy setting that we happen to handle very early on - 'RUNTIME_LINKED_LIBS', + 'MEMORY64', + 'INLINING_LIMIT', + 'DISABLE_EXCEPTION_CATCHING', + 'DISABLE_EXCEPTION_THROWING', + 'MAIN_MODULE', + 'SIDE_MODULE', + 'RELOCATABLE', + 'LINKABLE', + 'STRICT', + 'EMSCRIPTEN_TRACING', + 'PTHREADS', + 'USE_PTHREADS', # legacy name of PTHREADS setting + 'SHARED_MEMORY', + 'SUPPORT_LONGJMP', + 'WASM_OBJECT_FILES', + 'WASM_WORKERS', + 'BULK_MEMORY', + # Internal settings used during compilation + 'EXCEPTION_CATCHING_ALLOWED', + 'WASM_EXCEPTIONS', + 'LTO', + 'OPT_LEVEL', + 'DEBUG_LEVEL', + # Affects ports + 'GL_ENABLE_GET_PROC_ADDRESS', # NOTE: if SDL2 is updated to not rely on eglGetProcAddress(), this can be removed + # This is legacy setting that we happen to handle very early on + 'RUNTIME_LINKED_LIBS', }.union(PORTS_SETTINGS) # Unlike `LEGACY_SETTINGS`, deprecated settings can still be used @@ -117,19 +114,19 @@ # At some point in the future, once folks have stopped using these # settings we can move them to `LEGACY_SETTINGS`. DEPRECATED_SETTINGS = { - 'SUPPORT_ERRNO': 'emscripten no longer uses the setErrNo library function', - 'EXTRA_EXPORTED_RUNTIME_METHODS': 'please use EXPORTED_RUNTIME_METHODS instead', - 'DEMANGLE_SUPPORT': 'mangled names no longer appear in stack traces', - 'RUNTIME_LINKED_LIBS': 'you can simply list the libraries directly on the commandline now', - 'CLOSURE_WARNINGS': 'use -Wclosure instead', - 'LEGALIZE_JS_FFI': 'to disable JS type legalization use `-sWASM_BIGINT` or `-sSTANDALONE_WASM`', - 'ASYNCIFY_EXPORTS': 'please use JSPI_EXPORTS instead' + 'SUPPORT_ERRNO': 'emscripten no longer uses the setErrNo library function', + 'EXTRA_EXPORTED_RUNTIME_METHODS': 'please use EXPORTED_RUNTIME_METHODS instead', + 'DEMANGLE_SUPPORT': 'mangled names no longer appear in stack traces', + 'RUNTIME_LINKED_LIBS': 'you can simply list the libraries directly on the commandline now', + 'CLOSURE_WARNINGS': 'use -Wclosure instead', + 'LEGALIZE_JS_FFI': 'to disable JS type legalization use `-sWASM_BIGINT` or `-sSTANDALONE_WASM`', + 'ASYNCIFY_EXPORTS': 'please use JSPI_EXPORTS instead', } # Settings that don't need to be externalized when serializing to json because they # are not used by the JS compiler. INTERNAL_SETTINGS = { - 'SIDE_MODULE_IMPORTS', + 'SIDE_MODULE_IMPORTS', } user_settings: Dict[str, str] = {} @@ -204,7 +201,7 @@ def infer_types(self): def dict(self): return self.attrs - def external_dict(self, skip_keys={}): # noqa + def external_dict(self, skip_keys={}): # noqa external_settings = {k: v for k, v in self.dict().items() if k not in INTERNAL_SETTINGS and k not in skip_keys} # Only the names of the legacy settings are used by the JS compiler # so we can reduce the size of serialized json by simplifying this @@ -222,7 +219,9 @@ def limit_settings(self, allowed): def __getattr__(self, attr): if self.allowed_settings: - assert attr in self.allowed_settings, f"internal error: attempt to read setting '{attr}' while in limited settings mode" + assert ( + attr in self.allowed_settings + ), f"internal error: attempt to read setting '{attr}' while in limited settings mode" if attr in self.attrs: return self.attrs[attr] @@ -231,7 +230,9 @@ def __getattr__(self, attr): def __setattr__(self, name, value): if self.allowed_settings: - assert name in self.allowed_settings, f"internal error: attempt to write setting '{name}' while in limited settings mode" + assert ( + name in self.allowed_settings + ), f"internal error: attempt to write setting '{name}' while in limited settings mode" if name == 'STRICT' and value: for a in self.legacy_settings: diff --git a/tools/shared.py b/tools/shared.py index c6eb79378495e..287e6de4d17a5 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -100,7 +100,11 @@ # TODO(sbc): Investigate switching to shlex.quote def shlex_quote(arg): arg = os.fspath(arg) - if ' ' in arg and (not (arg.startswith('"') and arg.endswith('"'))) and (not (arg.startswith("'") and arg.endswith("'"))): + if ( + ' ' in arg + and (not (arg.startswith('"') and arg.endswith('"'))) + and (not (arg.startswith("'") and arg.endswith("'"))) + ): return '"' + arg.replace('"', '\\"') + '"' return arg @@ -154,10 +158,7 @@ def cap_max_workers_in_pool(max_workers): return max_workers -def run_multiple_processes(commands, - env=None, - route_stdout_to_temp_files_suffix=None, - cwd=None): +def run_multiple_processes(commands, env=None, route_stdout_to_temp_files_suffix=None, cwd=None): """Runs multiple subprocess commands. route_stdout_to_temp_files_suffix : string @@ -220,7 +221,10 @@ def get_finished_process(): idx = get_finished_process() finished_process = processes.pop(idx) if finished_process.returncode != 0: - exit_with_error('subprocess %d/%d failed (%s)! (cmdline: %s)' % (idx + 1, len(commands), returncode_to_str(finished_process.returncode), shlex_join(commands[idx]))) + exit_with_error( + 'subprocess %d/%d failed (%s)! (cmdline: %s)' + % (idx + 1, len(commands), returncode_to_str(finished_process.returncode), shlex_join(commands[idx])) + ) num_completed += 1 if route_stdout_to_temp_files_suffix: @@ -269,7 +273,9 @@ def get_npm_cmd(name): else: cmd = config.NODE_JS + [path_from_root('node_modules/.bin', name)] if not os.path.exists(cmd[-1]): - exit_with_error(f'{name} was not found! Please run "npm install" in Emscripten root directory to set up npm dependencies') + exit_with_error( + f'{name} was not found! Please run "npm install" in Emscripten root directory to set up npm dependencies' + ) return cmd @@ -308,7 +314,13 @@ def check_llvm_version(): if 'BUILDBOT_BUILDNUMBER' in os.environ: if actual.startswith('%d.' % (EXPECTED_LLVM_VERSION + 1)): return True - diagnostics.warning('version-check', 'LLVM version for clang executable "%s" appears incorrect (seeing "%s", expected "%s")', CLANG_CC, actual, EXPECTED_LLVM_VERSION) + diagnostics.warning( + 'version-check', + 'LLVM version for clang executable "%s" appears incorrect (seeing "%s", expected "%s")', + CLANG_CC, + actual, + EXPECTED_LLVM_VERSION, + ) return False @@ -422,7 +434,12 @@ def check_node(): try: run_process(config.NODE_JS + ['-e', 'console.log("hello")'], stdout=PIPE) except Exception as e: - exit_with_error('the configured node executable (%s) does not seem to work, check the paths in %s (%s)', config.NODE_JS, config.EM_CONFIG, str(e)) + exit_with_error( + 'the configured node executable (%s) does not seem to work, check the paths in %s (%s)', + config.NODE_JS, + config.EM_CONFIG, + str(e), + ) def generate_sanity(): @@ -502,7 +519,7 @@ def sanity_is_correct(): # when force is set. if force: perform_sanity_checks() - return True # all is well + return True # all is well return False if sanity_is_correct(): @@ -578,11 +595,13 @@ def get_emscripten_temp_dir(): EMSCRIPTEN_TEMP_DIR = tempfile.mkdtemp(prefix='emscripten_temp_', dir=TEMP_DIR) if not DEBUG_SAVE: + def prepare_to_clean_temp(d): def clean_temp(): utils.delete_dir(d) atexit.register(clean_temp) + # this global var might change later prepare_to_clean_temp(EMSCRIPTEN_TEMP_DIR) return EMSCRIPTEN_TEMP_DIR @@ -611,7 +630,9 @@ def setup_temp_dirs(): try: safe_ensure_dirs(EMSCRIPTEN_TEMP_DIR) except Exception as e: - exit_with_error(str(e) + f'Could not create canonical temp dir. Check definition of TEMP_DIR in {config.EM_CONFIG}') + exit_with_error( + str(e) + f'Could not create canonical temp dir. Check definition of TEMP_DIR in {config.EM_CONFIG}' + ) # Since the canonical temp directory is, by definition, the same # between all processes that run in DEBUG mode we need to use a multi diff --git a/tools/system_libs.py b/tools/system_libs.py index e6bbf1c3435b1..03854a25b938e 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -26,13 +26,35 @@ logger = logging.getLogger('system_libs') # Files that are part of libsockets.a and so should be excluded from libc.a -LIBC_SOCKETS = ['socket.c', 'socketpair.c', 'shutdown.c', 'bind.c', 'connect.c', - 'listen.c', 'accept.c', 'getsockname.c', 'getpeername.c', 'send.c', - 'recv.c', 'sendto.c', 'recvfrom.c', 'sendmsg.c', 'recvmsg.c', - 'getsockopt.c', 'setsockopt.c', 'freeaddrinfo.c', - 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', - 'gethostbyname_r.c', 'gethostbyname2.c', 'gethostbyname2_r.c', - 'in6addr_any.c', 'in6addr_loopback.c', 'accept4.c'] +LIBC_SOCKETS = [ + 'socket.c', + 'socketpair.c', + 'shutdown.c', + 'bind.c', + 'connect.c', + 'listen.c', + 'accept.c', + 'getsockname.c', + 'getpeername.c', + 'send.c', + 'recv.c', + 'sendto.c', + 'recvfrom.c', + 'sendmsg.c', + 'recvmsg.c', + 'getsockopt.c', + 'setsockopt.c', + 'freeaddrinfo.c', + 'gethostbyaddr.c', + 'gethostbyaddr_r.c', + 'gethostbyname.c', + 'gethostbyname_r.c', + 'gethostbyname2.c', + 'gethostbyname2_r.c', + 'in6addr_any.c', + 'in6addr_loopback.c', + 'accept4.c', +] # Experimental: Setting EMCC_USE_NINJA will cause system libraries to get built with ninja rather # than simple subprocesses. The primary benefit here is that we get accurate dependency tracking. @@ -75,12 +97,16 @@ def clean_env(): # At least one port also uses autoconf (harfbuzz) so we also need to clear # CFLAGS/LDFLAGS which we don't want to effect the inner call to configure. safe_env = os.environ.copy() - for opt in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS', - 'EMCC_CFLAGS', - 'EMCC_AUTODEBUG', - 'EMCC_FORCE_STDLIBS', - 'EMCC_ONLY_FORCED_STDLIBS', - 'EMMAKEN_JUST_CONFIGURE']: + for opt in [ + 'CFLAGS', + 'CXXFLAGS', + 'LDFLAGS', + 'EMCC_CFLAGS', + 'EMCC_AUTODEBUG', + 'EMCC_FORCE_STDLIBS', + 'EMCC_ONLY_FORCED_STDLIBS', + 'EMMAKEN_JUST_CONFIGURE', + ]: if opt in safe_env: del safe_env[opt] return safe_env @@ -429,8 +455,7 @@ def build(self, deterministic_paths=False): def generate(self): self.deterministic_paths = False - return cache.get(self.get_path(), self.do_generate, force=USE_NINJA == 2, quiet=USE_NINJA, - deferred=True) + return cache.get(self.get_path(), self.do_generate, force=USE_NINJA == 2, quiet=USE_NINJA, deferred=True) def get_link_flag(self): """ @@ -472,13 +497,17 @@ def generate_ninja(self, build_dir, libname): if self.deterministic_paths: source_dir = utils.path_from_root() relative_source_dir = os.path.relpath(source_dir, build_dir) - cflags += [f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', - f'-ffile-prefix-map={relative_source_dir}/=', - '-fdebug-compilation-dir=/emsdk/emscripten'] + cflags += [ + f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', + f'-ffile-prefix-map={relative_source_dir}/=', + '-fdebug-compilation-dir=/emsdk/emscripten', + ] asflags = get_base_cflags(preprocess=False) input_files = self.get_files() ninja_file = os.path.join(build_dir, 'build.ninja') - create_ninja_file(input_files, ninja_file, libname, cflags, asflags=asflags, customize_build_flags=self.customize_build_cmd) + create_ninja_file( + input_files, ninja_file, libname, cflags, asflags=asflags, customize_build_flags=self.customize_build_cmd + ) def build_objects(self, build_dir): """ @@ -497,8 +526,7 @@ def build_objects(self, build_dir): if batch_inputs: relative_source_dir = os.path.relpath(source_dir, build_dir) cflags += [f'-ffile-prefix-map={relative_source_dir}/='] - cflags += [f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', - '-fdebug-compilation-dir=/emsdk/emscripten'] + cflags += [f'-ffile-prefix-map={source_dir}=/emsdk/emscripten', '-fdebug-compilation-dir=/emsdk/emscripten'] case_insensitive = is_case_insensitive(build_dir) for src in self.get_files(): ext = shared.suffix(src) @@ -552,7 +580,7 @@ def build_objects(self, build_dir): for cmd, srcs in batches.items(): cmd = list(cmd) for i in range(0, len(srcs), chunk_size): - chunk_srcs = srcs[i:i + chunk_size] + chunk_srcs = srcs[i : i + chunk_size] commands.append(building.get_command_with_possible_response_file(cmd + chunk_srcs)) run_build_commands(commands, num_inputs=len(objects), build_dir=build_dir) @@ -657,8 +685,7 @@ def variations(cls): the behaviour. """ vary_on = cls.vary_on() - return [dict(zip(vary_on, toggles)) for toggles in - itertools.product([False, True], repeat=len(vary_on))] + return [dict(zip(vary_on, toggles)) for toggles in itertools.product([False, True], repeat=len(vary_on))] @classmethod def get_default_variation(cls, **kwargs): @@ -743,9 +770,7 @@ def vary_on(cls): @classmethod def get_default_variation(cls, **kwargs): return super().get_default_variation( - is_mt=settings.PTHREADS, - is_ww=settings.SHARED_MEMORY and not settings.PTHREADS, - **kwargs + is_mt=settings.PTHREADS, is_ww=settings.SHARED_MEMORY and not settings.PTHREADS, **kwargs ) @classmethod @@ -795,6 +820,7 @@ class Exceptions(IntEnum): is meant to be fast. You need to use a VM that has the EH support to use this. This is not fully working yet and still experimental. """ + NONE = auto() EMSCRIPTEN = auto() WASM = auto() @@ -828,9 +854,11 @@ def get_base_name(self): @classmethod def variations(cls, **kwargs): # noqa combos = super().variations() - return ([dict(eh_mode=Exceptions.NONE, **combo) for combo in combos] + - [dict(eh_mode=Exceptions.EMSCRIPTEN, **combo) for combo in combos] + - [dict(eh_mode=Exceptions.WASM, **combo) for combo in combos]) + return ( + [dict(eh_mode=Exceptions.NONE, **combo) for combo in combos] + + [dict(eh_mode=Exceptions.EMSCRIPTEN, **combo) for combo in combos] + + [dict(eh_mode=Exceptions.WASM, **combo) for combo in combos] + ) @classmethod def get_default_variation(cls, **kwargs): @@ -854,9 +882,7 @@ def get_cflags(self): if self.is_wasm: # DISABLE_EXCEPTION_THROWING=0 is the default, which is for Emscripten # EH/SjLj, so we should reverse it. - cflags += ['-sSUPPORT_LONGJMP=wasm', - '-sDISABLE_EXCEPTION_THROWING', - '-D__USING_WASM_SJLJ__'] + cflags += ['-sSUPPORT_LONGJMP=wasm', '-sDISABLE_EXCEPTION_THROWING', '-D__USING_WASM_SJLJ__'] return cflags def get_base_name(self): @@ -957,15 +983,16 @@ class libcompiler_rt(MTLibrary, SjLjLibrary): ] src_files = glob_in_path(src_dir, '*.c', excludes=excludes) src_files += files_in_path( - path='system/lib/compiler-rt', - filenames=[ - 'stack_ops.S', - 'stack_limits.S', - 'emscripten_setjmp.c', - 'emscripten_exception_builtins.c', - 'emscripten_tempret.s', - '__trap.c', - ]) + path='system/lib/compiler-rt', + filenames=[ + 'stack_ops.S', + 'stack_limits.S', + 'emscripten_setjmp.c', + 'emscripten_exception_builtins.c', + 'emscripten_tempret.s', + '__trap.c', + ], + ) class libnoexit(Library): @@ -974,10 +1001,7 @@ class libnoexit(Library): src_files = ['atexit_dummy.c'] -class libc(MuslInternalLibrary, - DebugLibrary, - AsanInstrumentedLibrary, - MTLibrary): +class libc(MuslInternalLibrary, DebugLibrary, AsanInstrumentedLibrary, MTLibrary): name = 'libc' # Without -fno-builtin, LLVM can optimize away or convert calls to library @@ -990,18 +1014,20 @@ class libc(MuslInternalLibrary, cflags = ['-Os', '-fno-inline-functions', '-fno-builtin'] # Disable certain warnings for code patterns that are contained in upstream musl - cflags += ['-Wno-ignored-attributes', - # tre.h defines NDEBUG internally itself - '-Wno-macro-redefined', - '-Wno-shift-op-parentheses', - '-Wno-string-plus-int', - '-Wno-missing-braces', - '-Wno-logical-op-parentheses', - '-Wno-bitwise-op-parentheses', - '-Wno-unused-but-set-variable', - '-Wno-unused-variable', - '-Wno-unused-label', - '-Wno-pointer-sign'] + cflags += [ + '-Wno-ignored-attributes', + # tre.h defines NDEBUG internally itself + '-Wno-macro-redefined', + '-Wno-shift-op-parentheses', + '-Wno-string-plus-int', + '-Wno-missing-braces', + '-Wno-logical-op-parentheses', + '-Wno-bitwise-op-parentheses', + '-Wno-unused-but-set-variable', + '-Wno-unused-variable', + '-Wno-unused-label', + '-Wno-pointer-sign', + ] def __init__(self, **kwargs): self.non_lto_files = self.get_libcall_files() @@ -1022,38 +1048,67 @@ def get_libcall_files(self): # functions - fmin uses signbit, for example, so signbit must be here (so if # fmin is added by codegen, it will have all it needs). math_files = [ - 'fmin.c', 'fminf.c', 'fminl.c', - 'fmax.c', 'fmaxf.c', 'fmaxl.c', - 'fmod.c', 'fmodf.c', 'fmodl.c', - 'logf.c', 'logf_data.c', - 'log2f.c', 'log2f_data.c', - 'log10.c', 'log10f.c', - 'exp.c', 'exp_data.c', + 'fmin.c', + 'fminf.c', + 'fminl.c', + 'fmax.c', + 'fmaxf.c', + 'fmaxl.c', + 'fmod.c', + 'fmodf.c', + 'fmodl.c', + 'logf.c', + 'logf_data.c', + 'log2f.c', + 'log2f_data.c', + 'log10.c', + 'log10f.c', + 'exp.c', + 'exp_data.c', 'exp2.c', - 'exp2f.c', 'exp2f_data.c', - 'exp10.c', 'exp10f.c', - 'ldexp.c', 'ldexpf.c', 'ldexpl.c', - 'scalbn.c', '__fpclassifyl.c', - '__signbitl.c', '__signbitf.c', '__signbit.c', - '__math_divzero.c', '__math_divzerof.c', - '__math_oflow.c', '__math_oflowf.c', - '__math_uflow.c', '__math_uflowf.c', - '__math_invalid.c', '__math_invalidf.c', '__math_invalidl.c', - 'pow.c', 'pow_data.c', 'log.c', 'log_data.c', 'log2.c', 'log2_data.c', + 'exp2f.c', + 'exp2f_data.c', + 'exp10.c', + 'exp10f.c', + 'ldexp.c', + 'ldexpf.c', + 'ldexpl.c', + 'scalbn.c', + '__fpclassifyl.c', + '__signbitl.c', + '__signbitf.c', + '__signbit.c', + '__math_divzero.c', + '__math_divzerof.c', + '__math_oflow.c', + '__math_oflowf.c', + '__math_uflow.c', + '__math_uflowf.c', + '__math_invalid.c', + '__math_invalidf.c', + '__math_invalidl.c', + 'pow.c', + 'pow_data.c', + 'log.c', + 'log_data.c', + 'log2.c', + 'log2_data.c', 'scalbnf.c', ] math_files = files_in_path(path='system/lib/libc/musl/src/math', filenames=math_files) - exit_files = files_in_path( - path='system/lib/libc/musl/src/exit', - filenames=['atexit.c']) + exit_files = files_in_path(path='system/lib/libc/musl/src/exit', filenames=['atexit.c']) other_files = files_in_path( path='system/lib/libc', - filenames=['emscripten_memcpy.c', 'emscripten_memset.c', - 'emscripten_scan_stack.c', - 'emscripten_get_heap_size.c', # needed by malloc - 'emscripten_memmove.c']) + filenames=[ + 'emscripten_memcpy.c', + 'emscripten_memset.c', + 'emscripten_scan_stack.c', + 'emscripten_get_heap_size.c', # needed by malloc + 'emscripten_memmove.c', + ], + ) # Calls to iprintf can be generated during codegen. Ideally we wouldn't # compile these with -O2 like we do the rest of compiler-rt since its # probably not performance sensitive. However we don't currently have @@ -1061,21 +1116,27 @@ def get_libcall_files(self): # move all this stuff back into libc once we it LTO compatible. iprintf_files = files_in_path( path='system/lib/libc/musl/src/stdio', - filenames=['__towrite.c', '__overflow.c', 'fwrite.c', 'fputs.c', - 'getc.c', - 'fputc.c', - 'fgets.c', - 'putc.c', 'putc_unlocked.c', - 'putchar.c', 'putchar_unlocked.c', - 'printf.c', 'puts.c', '__lockfile.c']) - iprintf_files += files_in_path( - path='system/lib/libc/musl/src/string', - filenames=['strlen.c']) + filenames=[ + '__towrite.c', + '__overflow.c', + 'fwrite.c', + 'fputs.c', + 'getc.c', + 'fputc.c', + 'fgets.c', + 'putc.c', + 'putc_unlocked.c', + 'putchar.c', + 'putchar_unlocked.c', + 'printf.c', + 'puts.c', + '__lockfile.c', + ], + ) + iprintf_files += files_in_path(path='system/lib/libc/musl/src/string', filenames=['strlen.c']) # Transitively required by many system call imports - errno_files = files_in_path( - path='system/lib/libc/musl/src/errno', - filenames=['__errno_location.c', 'strerror.c']) + errno_files = files_in_path(path='system/lib/libc/musl/src/errno', filenames=['__errno_location.c', 'strerror.c']) return math_files + exit_files + other_files + iprintf_files + errno_files @@ -1085,26 +1146,54 @@ def get_files(self): # musl modules ignore = [ - 'ipc', 'passwd', 'signal', 'sched', 'time', 'linux', - 'aio', 'exit', 'legacy', 'mq', 'setjmp', - 'ldso', 'malloc' + 'ipc', + 'passwd', + 'signal', + 'sched', + 'time', + 'linux', + 'aio', + 'exit', + 'legacy', + 'mq', + 'setjmp', + 'ldso', + 'malloc', ] # individual files ignore += [ - 'memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', - 'res_query.c', 'res_querydomain.c', - 'proto.c', - 'ppoll.c', - 'syscall.c', 'popen.c', 'pclose.c', - 'getgrouplist.c', 'initgroups.c', 'wordexp.c', 'timer_create.c', - 'getauxval.c', - 'lookup_name.c', - # 'process' exclusion - 'fork.c', 'vfork.c', 'posix_spawn.c', 'posix_spawnp.c', 'execve.c', 'waitid.c', 'system.c', - '_Fork.c', - # 'env' exclusion - '__reset_tls.c', '__init_tls.c', '__libc_start_main.c', + 'memcpy.c', + 'memset.c', + 'memmove.c', + 'getaddrinfo.c', + 'getnameinfo.c', + 'res_query.c', + 'res_querydomain.c', + 'proto.c', + 'ppoll.c', + 'syscall.c', + 'popen.c', + 'pclose.c', + 'getgrouplist.c', + 'initgroups.c', + 'wordexp.c', + 'timer_create.c', + 'getauxval.c', + 'lookup_name.c', + # 'process' exclusion + 'fork.c', + 'vfork.c', + 'posix_spawn.c', + 'posix_spawnp.c', + 'execve.c', + 'waitid.c', + 'system.c', + '_Fork.c', + # 'env' exclusion + '__reset_tls.c', + '__init_tls.c', + '__libc_start_main.c', ] ignore += LIBC_SOCKETS @@ -1113,12 +1202,16 @@ def get_files(self): ignore += [ 'clone.c', 'pthread_create.c', - 'pthread_kill.c', 'pthread_sigmask.c', - '__set_thread_area.c', 'synccall.c', - '__syscall_cp.c', '__tls_get_addr.c', + 'pthread_kill.c', + 'pthread_sigmask.c', + '__set_thread_area.c', + 'synccall.c', + '__syscall_cp.c', + '__tls_get_addr.c', '__unmapself.c', # Empty files, simply ignore them. - 'syscall_cp.c', 'tls.c', + 'syscall_cp.c', + 'tls.c', # TODO: Support these. See #12216. 'pthread_setname_np.c', 'pthread_getname_np.c', @@ -1138,7 +1231,8 @@ def get_files(self): 'emscripten_futex_wait.c', 'emscripten_futex_wake.c', 'emscripten_yield.c', - ]) + ], + ) else: ignore += ['thread'] libc_files += files_in_path( @@ -1185,19 +1279,19 @@ def get_files(self): 'thrd_join.c', 'thrd_sleep.c', 'thrd_yield.c', - ]) + ], + ) libc_files += files_in_path( path='system/lib/pthread', filenames=[ 'library_pthread_stub.c', 'pthread_self_stub.c', 'proxying_stub.c', - ]) + ], + ) # These files are in libc directories, but only built in libc_optz. - ignore += [ - 'pow_small.c', 'log_small.c', 'log2_small.c' - ] + ignore += ['pow_small.c', 'log_small.c', 'log2_small.c'] ignore = set(ignore) for dirpath, dirnames, filenames in os.walk(musl_srcdir): @@ -1212,106 +1306,107 @@ def get_files(self): # Allowed files from ignored modules libc_files += files_in_path( - path='system/lib/libc/musl/src/time', - filenames=[ - 'clock_settime.c', - 'asctime_r.c', - 'asctime.c', - 'ctime.c', - 'difftime.c', - 'ftime.c', - 'gmtime.c', - 'localtime.c', - 'nanosleep.c', - 'clock.c', - 'clock_nanosleep.c', - 'clock_getres.c', - 'clock_gettime.c', - 'ctime_r.c', - 'timespec_get.c', - 'utime.c', - '__map_file.c', - 'strftime.c', - '__tz.c', - '__tm_to_secs.c', - '__year_to_secs.c', - '__month_to_secs.c', - 'wcsftime.c', - ]) + path='system/lib/libc/musl/src/time', + filenames=[ + 'clock_settime.c', + 'asctime_r.c', + 'asctime.c', + 'ctime.c', + 'difftime.c', + 'ftime.c', + 'gmtime.c', + 'localtime.c', + 'nanosleep.c', + 'clock.c', + 'clock_nanosleep.c', + 'clock_getres.c', + 'clock_gettime.c', + 'ctime_r.c', + 'timespec_get.c', + 'utime.c', + '__map_file.c', + 'strftime.c', + '__tz.c', + '__tm_to_secs.c', + '__year_to_secs.c', + '__month_to_secs.c', + 'wcsftime.c', + ], + ) libc_files += files_in_path( - path='system/lib/libc/musl/src/legacy', - filenames=['getpagesize.c', 'err.c', 'euidaccess.c']) + path='system/lib/libc/musl/src/legacy', filenames=['getpagesize.c', 'err.c', 'euidaccess.c'] + ) libc_files += files_in_path( - path='system/lib/libc/musl/src/linux', - filenames=['getdents.c', 'gettid.c', 'utimes.c', 'statx.c', 'wait4.c', 'wait3.c']) + path='system/lib/libc/musl/src/linux', + filenames=['getdents.c', 'gettid.c', 'utimes.c', 'statx.c', 'wait4.c', 'wait3.c'], + ) - libc_files += files_in_path( - path='system/lib/libc/musl/src/sched', - filenames=['sched_yield.c']) + libc_files += files_in_path(path='system/lib/libc/musl/src/sched', filenames=['sched_yield.c']) libc_files += files_in_path( - path='system/lib/libc/musl/src/exit', - filenames=['abort.c', '_Exit.c', 'atexit.c', 'at_quick_exit.c', 'quick_exit.c']) + path='system/lib/libc/musl/src/exit', + filenames=['abort.c', '_Exit.c', 'atexit.c', 'at_quick_exit.c', 'quick_exit.c'], + ) libc_files += files_in_path( - path='system/lib/libc/musl/src/ldso', - filenames=['dladdr.c', 'dlerror.c', 'dlsym.c', 'dlclose.c']) + path='system/lib/libc/musl/src/ldso', filenames=['dladdr.c', 'dlerror.c', 'dlsym.c', 'dlclose.c'] + ) libc_files += files_in_path( - path='system/lib/libc/musl/src/signal', - filenames=[ - 'block.c', - 'getitimer.c', - 'killpg.c', - 'setitimer.c', - 'sigorset.c', - 'sigandset.c', - 'sigaddset.c', - 'sigdelset.c', - 'sigemptyset.c', - 'sigisemptyset.c', - 'sigfillset.c', - 'sigismember.c', - 'siginterrupt.c', - 'signal.c', - 'sigprocmask.c', - 'sigrtmax.c', - 'sigrtmin.c', - 'sigwait.c', - 'sigwaitinfo.c', - ]) + path='system/lib/libc/musl/src/signal', + filenames=[ + 'block.c', + 'getitimer.c', + 'killpg.c', + 'setitimer.c', + 'sigorset.c', + 'sigandset.c', + 'sigaddset.c', + 'sigdelset.c', + 'sigemptyset.c', + 'sigisemptyset.c', + 'sigfillset.c', + 'sigismember.c', + 'siginterrupt.c', + 'signal.c', + 'sigprocmask.c', + 'sigrtmax.c', + 'sigrtmin.c', + 'sigwait.c', + 'sigwaitinfo.c', + ], + ) libc_files += files_in_path( - path='system/lib/libc', - filenames=[ - 'emscripten_console.c', - 'emscripten_fiber.c', - 'emscripten_get_heap_size.c', - 'emscripten_memcpy.c', - 'emscripten_memmove.c', - 'emscripten_memset.c', - 'emscripten_mmap.c', - 'emscripten_scan_stack.c', - 'emscripten_time.c', - 'mktime.c', - 'kill.c', - 'lookup_name.c', - 'pthread_sigmask.c', - 'raise.c', - 'sigaction.c', - 'sigtimedwait.c', - 'wasi-helpers.c', - 'system.c', - ]) + path='system/lib/libc', + filenames=[ + 'emscripten_console.c', + 'emscripten_fiber.c', + 'emscripten_get_heap_size.c', + 'emscripten_memcpy.c', + 'emscripten_memmove.c', + 'emscripten_memset.c', + 'emscripten_mmap.c', + 'emscripten_scan_stack.c', + 'emscripten_time.c', + 'mktime.c', + 'kill.c', + 'lookup_name.c', + 'pthread_sigmask.c', + 'raise.c', + 'sigaction.c', + 'sigtimedwait.c', + 'wasi-helpers.c', + 'system.c', + ], + ) if settings.RELOCATABLE: libc_files += files_in_path(path='system/lib/libc', filenames=['dynlink.c']) - libc_files += files_in_path( - path='system/lib/pthread', - filenames=['thread_profiler.c']) + libc_files += files_in_path(path='system/lib/pthread', filenames=['thread_profiler.c']) libc_files += glob_in_path('system/lib/libc/compat', '*.c') @@ -1354,14 +1449,13 @@ def get_libcall_files(self): # some files also appear in libc, and a #define affects them mem_files = files_in_path( - path='system/lib/libc', - filenames=['emscripten_memcpy.c', 'emscripten_memset.c', - 'emscripten_memmove.c']) + path='system/lib/libc', filenames=['emscripten_memcpy.c', 'emscripten_memset.c', 'emscripten_memmove.c'] + ) # some functions have separate files math_files = files_in_path( - path='system/lib/libc/musl/src/math', - filenames=['pow_small.c', 'log_small.c', 'log2_small.c']) + path='system/lib/libc/musl/src/math', filenames=['pow_small.c', 'log_small.c', 'log2_small.c'] + ) return mem_files + math_files @@ -1369,9 +1463,7 @@ def get_files(self): libcall_files = self.get_libcall_files() # some files also appear in libc, and a #define affects them - mem_files = files_in_path( - path='system/lib/libc/musl/src/string', - filenames=['memcmp.c']) + mem_files = files_in_path(path='system/lib/libc/musl/src/string', filenames=['memcmp.c']) return libcall_files + mem_files @@ -1389,15 +1481,23 @@ def can_use(self): # EMCC_FORCE_STDLIBS can have a similar effect of forcing all libraries. # In both cases, the build is not one that is hyper-focused on code size, # and so optz is not that important. - return super(libc_optz, self).can_use() and settings.SHRINK_LEVEL >= 2 and \ - not settings.LINKABLE and not os.environ.get('EMCC_FORCE_STDLIBS') + return ( + super(libc_optz, self).can_use() + and settings.SHRINK_LEVEL >= 2 + and not settings.LINKABLE + and not os.environ.get('EMCC_FORCE_STDLIBS') + ) class libbulkmemory(MuslInternalLibrary, AsanInstrumentedLibrary): name = 'libbulkmemory' src_dir = 'system/lib/libc' - src_files = ['emscripten_memcpy.c', 'emscripten_memset.c', - 'emscripten_memcpy_bulkmem.S', 'emscripten_memset_bulkmem.S'] + src_files = [ + 'emscripten_memcpy.c', + 'emscripten_memset.c', + 'emscripten_memcpy_bulkmem.S', + 'emscripten_memset_bulkmem.S', + ] cflags = ['-mbulk-memory'] def can_use(self): @@ -1409,9 +1509,7 @@ class libprintf_long_double(libc): cflags = ['-DEMSCRIPTEN_PRINTF_LONG_DOUBLE'] def get_files(self): - return files_in_path( - path='system/lib/libc/musl/src/stdio', - filenames=['vfprintf.c']) + return files_in_path(path='system/lib/libc/musl/src/stdio', filenames=['vfprintf.c']) def can_use(self): return super(libprintf_long_double, self).can_use() and settings.PRINTF_LONG_DOUBLE @@ -1461,22 +1559,17 @@ def get_default_variation(cls, **kwargs): def get_files(self): files = [] if self.is_stub: - files = [ - 'library_wasm_worker_stub.c' - ] + files = ['library_wasm_worker_stub.c'] else: files = [ 'library_wasm_worker.c', 'wasm_worker_initialize.S', ] - return files_in_path( - path='system/lib/wasm_worker', - filenames=files) + return files_in_path(path='system/lib/wasm_worker', filenames=files) def can_use(self): # see src/library_wasm_worker.js - return super().can_use() and not settings.SINGLE_FILE \ - and not settings.RELOCATABLE and not settings.PROXY_TO_WORKER + return super().can_use() and not settings.SINGLE_FILE and not settings.RELOCATABLE and not settings.PROXY_TO_WORKER class libsockets(MuslInternalLibrary, MTLibrary): @@ -1485,9 +1578,7 @@ class libsockets(MuslInternalLibrary, MTLibrary): cflags = ['-Os', '-fno-inline-functions', '-fno-builtin', '-Wno-shift-op-parentheses'] def get_files(self): - return files_in_path( - path='system/lib/libc/musl/src/network', - filenames=LIBC_SOCKETS) + return files_in_path(path='system/lib/libc/musl/src/network', filenames=LIBC_SOCKETS) def can_use(self): return super(libsockets, self).can_use() and not settings.PROXY_POSIX_SOCKETS @@ -1565,13 +1656,13 @@ def can_use(self): class libcxxabi(NoExceptLibrary, MTLibrary, DebugLibrary): name = 'libc++abi' cflags = [ - '-Oz', - '-fno-inline-functions', - '-D_LIBCPP_BUILDING_LIBRARY', - '-D_LIBCXXABI_BUILDING_LIBRARY', - '-DLIBCXXABI_NON_DEMANGLING_TERMINATE', - '-std=c++20', - ] + '-Oz', + '-fno-inline-functions', + '-D_LIBCPP_BUILDING_LIBRARY', + '-D_LIBCXXABI_BUILDING_LIBRARY', + '-DLIBCXXABI_NON_DEMANGLING_TERMINATE', + '-std=c++20', + ] includes = ['system/lib/libcxx/src'] def __init__(self, **kwargs): @@ -1621,17 +1712,11 @@ def get_files(self): elif self.eh_mode == Exceptions.EMSCRIPTEN: filenames += ['cxa_exception_emscripten.cpp'] elif self.eh_mode == Exceptions.WASM: - filenames += [ - 'cxa_exception_storage.cpp', - 'cxa_exception.cpp', - 'cxa_personality.cpp' - ] + filenames += ['cxa_exception_storage.cpp', 'cxa_exception.cpp', 'cxa_personality.cpp'] else: assert False - return files_in_path( - path='system/lib/libcxxabi/src', - filenames=filenames) + return files_in_path(path='system/lib/libcxxabi/src', filenames=filenames) class libcxx(NoExceptLibrary, MTLibrary): @@ -1720,8 +1805,19 @@ class libmalloc(MTLibrary): def __init__(self, **kwargs): self.malloc = kwargs.pop('malloc') - if self.malloc not in ('dlmalloc', 'emmalloc', 'emmalloc-debug', 'emmalloc-memvalidate', 'emmalloc-verbose', 'emmalloc-memvalidate-verbose', 'mimalloc', 'none'): - raise Exception('malloc must be one of "emmalloc[-debug|-memvalidate][-verbose]", "mimalloc", "dlmalloc" or "none", see settings.js') + if self.malloc not in ( + 'dlmalloc', + 'emmalloc', + 'emmalloc-debug', + 'emmalloc-memvalidate', + 'emmalloc-verbose', + 'emmalloc-memvalidate-verbose', + 'mimalloc', + 'none', + ): + raise Exception( + 'malloc must be one of "emmalloc[-debug|-memvalidate][-verbose]", "mimalloc", "dlmalloc" or "none", see settings.js' + ) self.is_tracing = kwargs.pop('is_tracing') self.memvalidate = kwargs.pop('memvalidate') @@ -1732,9 +1828,13 @@ def __init__(self, **kwargs): def get_files(self): malloc_base = self.malloc.replace('-memvalidate', '').replace('-verbose', '').replace('-debug', '') - malloc = utils.path_from_root('system/lib', { - 'dlmalloc': 'dlmalloc.c', 'emmalloc': 'emmalloc.c', - }[malloc_base]) + malloc = utils.path_from_root( + 'system/lib', + { + 'dlmalloc': 'dlmalloc.c', + 'emmalloc': 'emmalloc.c', + }[malloc_base], + ) # Include sbrk.c in libc, it uses tracing and libc itself doesn't have a tracing variant. sbrk = utils.path_from_root('system/lib/libc/sbrk.c') return [malloc, sbrk] @@ -1779,17 +1879,27 @@ def get_default_variation(cls, **kwargs): is_tracing=settings.EMSCRIPTEN_TRACING, memvalidate='memvalidate' in settings.MALLOC, verbose='verbose' in settings.MALLOC, - **kwargs + **kwargs, ) @classmethod def variations(cls): combos = super().variations() - return ([dict(malloc='dlmalloc', **combo) for combo in combos if not combo['memvalidate'] and not combo['verbose']] + - [dict(malloc='emmalloc', **combo) for combo in combos if not combo['memvalidate'] and not combo['verbose']] + - [dict(malloc='emmalloc-memvalidate-verbose', **combo) for combo in combos if combo['memvalidate'] and combo['verbose']] + - [dict(malloc='emmalloc-memvalidate', **combo) for combo in combos if combo['memvalidate'] and not combo['verbose']] + - [dict(malloc='emmalloc-verbose', **combo) for combo in combos if combo['verbose'] and not combo['memvalidate']]) + return ( + [dict(malloc='dlmalloc', **combo) for combo in combos if not combo['memvalidate'] and not combo['verbose']] + + [dict(malloc='emmalloc', **combo) for combo in combos if not combo['memvalidate'] and not combo['verbose']] + + [ + dict(malloc='emmalloc-memvalidate-verbose', **combo) + for combo in combos + if combo['memvalidate'] and combo['verbose'] + ] + + [ + dict(malloc='emmalloc-memvalidate', **combo) + for combo in combos + if combo['memvalidate'] and not combo['verbose'] + ] + + [dict(malloc='emmalloc-verbose', **combo) for combo in combos if combo['verbose'] and not combo['memvalidate']] + ) class libmimalloc(MTLibrary): @@ -1827,7 +1937,7 @@ class libmimalloc(MTLibrary): path='system/lib/mimalloc/src', glob_pattern='*.c', # mimalloc includes some files at the source level, so exclude them here. - excludes=['alloc-override.c', 'free.c', 'page-queue.c', 'static.c'] + excludes=['alloc-override.c', 'free.c', 'page-queue.c', 'static.c'], ) src_files += [utils.path_from_root('system/lib/mimalloc/src/prim/prim.c')] src_files += [utils.path_from_root('system/lib/emmalloc.c')] @@ -1901,7 +2011,7 @@ def get_default_variation(cls, **kwargs): is_ofb=settings.OFFSCREEN_FRAMEBUFFER, is_full_es3=settings.FULL_ES3, is_enable_get_proc_address=settings.GL_ENABLE_GET_PROC_ADDRESS, - **kwargs + **kwargs, ) @@ -2004,24 +2114,30 @@ def get_default_variation(cls, **kwargs): def get_files(self): backends = files_in_path( - path='system/lib/wasmfs/backends', - filenames=['fetch_backend.cpp', - 'ignore_case_backend.cpp', - 'js_file_backend.cpp', - 'memory_backend.cpp', - 'node_backend.cpp', - 'opfs_backend.cpp']) + path='system/lib/wasmfs/backends', + filenames=[ + 'fetch_backend.cpp', + 'ignore_case_backend.cpp', + 'js_file_backend.cpp', + 'memory_backend.cpp', + 'node_backend.cpp', + 'opfs_backend.cpp', + ], + ) return backends + files_in_path( - path='system/lib/wasmfs', - filenames=['file.cpp', - 'file_table.cpp', - 'js_api.cpp', - 'emscripten.cpp', - 'paths.cpp', - 'special_files.cpp', - 'support.cpp', - 'syscalls.cpp', - 'wasmfs.cpp']) + path='system/lib/wasmfs', + filenames=[ + 'file.cpp', + 'file_table.cpp', + 'js_api.cpp', + 'emscripten.cpp', + 'paths.cpp', + 'special_files.cpp', + 'support.cpp', + 'syscalls.cpp', + 'wasmfs.cpp', + ], + ) def can_use(self): return settings.WASMFS @@ -2049,9 +2165,7 @@ class libwasmfs_noderawfs(Library): includes = ['system/lib/wasmfs'] def get_files(self): - return files_in_path( - path='system/lib/wasmfs/backends', - filenames=['noderawfs_root.cpp']) + return files_in_path(path='system/lib/wasmfs/backends', filenames=['noderawfs_root.cpp']) def can_use(self): return settings.WASMFS and settings.NODERAWFS @@ -2085,9 +2199,7 @@ class libubsan_minimal_rt(CompilerRTLibrary, MTLibrary): class libsanitizer_common_rt(CompilerRTLibrary, MTLibrary): name = 'libsanitizer_common_rt' # TODO(sbc): We should not need musl-internal headers here. - includes = ['system/lib/libc/musl/src/internal', - 'system/lib/compiler-rt/lib', - 'system/lib/libc'] + includes = ['system/lib/libc/musl/src/internal', 'system/lib/compiler-rt/lib', 'system/lib/libc'] never_force = True cflags = [ '-D_LARGEFILE64_SOURCE', @@ -2131,8 +2243,7 @@ class liblsan_rt(SanitizerLibrary): includes = ['system/lib/libc'] src_dir = 'system/lib/compiler-rt/lib/lsan' - src_glob_exclude = ['lsan_common.cpp', 'lsan_common_mac.cpp', 'lsan_common_linux.cpp', - 'lsan_common_emscripten.cpp'] + src_glob_exclude = ['lsan_common.cpp', 'lsan_common_mac.cpp', 'lsan_common_linux.cpp', 'lsan_common_emscripten.cpp'] class libasan_rt(SanitizerLibrary): @@ -2202,35 +2313,32 @@ def get_default_variation(cls, **kwargs): is_mem_grow=settings.ALLOW_MEMORY_GROWTH, is_pure=settings.PURE_WASI, nocatch=settings.DISABLE_EXCEPTION_CATCHING and not settings.WASM_EXCEPTIONS, - **kwargs + **kwargs, ) def get_files(self): files = files_in_path( - path='system/lib/standalone', - filenames=['standalone.c', - 'standalone_wasm_stdio.c', - '__main_void.c']) - files += files_in_path( - path='system/lib/libc', - filenames=['emscripten_memcpy.c', 'emscripten_memset.c']) + path='system/lib/standalone', filenames=['standalone.c', 'standalone_wasm_stdio.c', '__main_void.c'] + ) + files += files_in_path(path='system/lib/libc', filenames=['emscripten_memcpy.c', 'emscripten_memset.c']) # It is more efficient to use JS methods for time, normally. files += files_in_path( - path='system/lib/libc/musl/src/time', - filenames=['__secs_to_tm.c', - '__tz.c', - 'gettimeofday.c', - 'localtime_r.c', - 'gmtime_r.c', - 'mktime.c', - 'strptime.c', - 'timegm.c', - 'time.c']) + path='system/lib/libc/musl/src/time', + filenames=[ + '__secs_to_tm.c', + '__tz.c', + 'gettimeofday.c', + 'localtime_r.c', + 'gmtime_r.c', + 'mktime.c', + 'strptime.c', + 'timegm.c', + 'time.c', + ], + ) # It is more efficient to use JS for __assert_fail, as it avoids always # including fprintf etc. - files += files_in_path( - path='system/lib/libc/musl/src/exit', - filenames=['assert.c', 'exit.c']) + files += files_in_path(path='system/lib/libc/musl/src/exit', filenames=['assert.c', 'exit.c']) return files def can_use(self): @@ -2278,7 +2386,9 @@ def get_libs_to_link(args): if only_forced: # One of the purposes EMCC_ONLY_FORCED_STDLIBS was to skip the scanning # of the input files for reverse dependencies. - diagnostics.warning('deprecated', 'EMCC_ONLY_FORCED_STDLIBS is deprecated. Use `-nostdlib` to avoid linking standard libraries') + diagnostics.warning( + 'deprecated', 'EMCC_ONLY_FORCED_STDLIBS is deprecated. Use `-nostdlib` to avoid linking standard libraries' + ) if force == '1': force_include = [name for name, lib in system_libs_map.items() if not lib.never_force] elif force is not None: @@ -2368,8 +2478,7 @@ def add_sanitizer_libs(): if settings.PRINTF_LONG_DOUBLE: add_library('libprintf_long_double') # See comment in libc_optz itself - if settings.SHRINK_LEVEL >= 2 and not settings.LINKABLE and \ - not os.environ.get('EMCC_FORCE_STDLIBS'): + if settings.SHRINK_LEVEL >= 2 and not settings.LINKABLE and not os.environ.get('EMCC_FORCE_STDLIBS'): add_library('libc_optz') if settings.BULK_MEMORY: add_library('libbulkmemory') @@ -2403,9 +2512,7 @@ def add_sanitizer_libs(): if settings.LINK_AS_CXX: add_library('libwebgpu_cpp') - if settings.WASM_WORKERS and (not settings.SINGLE_FILE and - not settings.RELOCATABLE and - not settings.PROXY_TO_WORKER): + if settings.WASM_WORKERS and (not settings.SINGLE_FILE and not settings.RELOCATABLE and not settings.PROXY_TO_WORKER): # When we include libwasm_workers we use `--whole-archive` to ensure # that the static constructor (`emscripten_wasm_worker_main_thread_initialize`) # is run. @@ -2427,7 +2534,6 @@ def add_sanitizer_libs(): def calculate(args): - libs_to_link = get_libs_to_link(args) # When LINKABLE is set the entire link command line is wrapped in --whole-archive by @@ -2503,12 +2609,15 @@ def install_system_headers(stamp): # Create a version header based on the emscripten-version.txt version_file = cache.get_include_dir('emscripten/version.h') - utils.write_file(version_file, textwrap.dedent(f'''\ + utils.write_file( + version_file, + textwrap.dedent(f'''\ /* Automatically generated by tools/system_libs.py */ #define __EMSCRIPTEN_major__ {utils.EMSCRIPTEN_VERSION_MAJOR} #define __EMSCRIPTEN_minor__ {utils.EMSCRIPTEN_VERSION_MINOR} #define __EMSCRIPTEN_tiny__ {utils.EMSCRIPTEN_VERSION_TINY} - ''')) + '''), + ) # Create a stamp file that signal that the headers have been installed # Removing this file, or running `emcc --clear-cache` or running diff --git a/tools/tempfiles.py b/tools/tempfiles.py index 48c98ed4a5316..69671ea8d407b 100644 --- a/tools/tempfiles.py +++ b/tools/tempfiles.py @@ -33,15 +33,17 @@ def get_file(self, suffix): 'with TempFiles.get_file(..) as filename:'. The file will be deleted immediately once the 'with' block is exited. """ + class TempFileObject: def __enter__(self_): self_.file = tempfile.NamedTemporaryFile(dir=self.tmpdir, suffix=suffix, delete=False) - self_.file.close() # NamedTemporaryFile passes out open file handles, but callers prefer filenames (and open their own handles manually if needed) + self_.file.close() # NamedTemporaryFile passes out open file handles, but callers prefer filenames (and open their own handles manually if needed) return self_.file.name def __exit__(self_, _type, _value, _traceback): if not self.save_debug_files: utils.delete_file(self_.file.name) + return TempFileObject() def get_dir(self): diff --git a/tools/toolchain_profiler.py b/tools/toolchain_profiler.py index 0bfc1c8d7121a..039fbdebf634b 100644 --- a/tools/toolchain_profiler.py +++ b/tools/toolchain_profiler.py @@ -121,7 +121,7 @@ class ToolchainProfiler: # Provide a running counter towards negative numbers for PIDs for which we # don't know what the actual process ID is imaginary_pid_ = 0 - profiler_logs_path = None # Log file not opened yet + profiler_logs_path = None # Log file not opened yet block_stack = [] @@ -144,7 +144,9 @@ def log_access(): # the parent->child process spawns for the subprocessing pools. Therefore # any profiling events that the subprocess children generate are virtually # treated as if they were performed by the parent PID. - return open(os.path.join(ToolchainProfiler.profiler_logs_path, 'toolchain_profiler.pid_' + str(os.getpid()) + '.json'), 'a') + return open( + os.path.join(ToolchainProfiler.profiler_logs_path, 'toolchain_profiler.pid_' + str(os.getpid()) + '.json'), 'a' + ) @staticmethod def escape_string(arg): @@ -171,7 +173,17 @@ def record_process_start(write_log_entry=True): if write_log_entry: with ToolchainProfiler.log_access() as f: - f.write('[\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"start","time":' + ToolchainProfiler.timestamp() + ',"cmdLine":["' + '","'.join(ToolchainProfiler.escape_args(sys.argv)) + '"]}') + f.write( + '[\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"start","time":' + + ToolchainProfiler.timestamp() + + ',"cmdLine":["' + + '","'.join(ToolchainProfiler.escape_args(sys.argv)) + + '"]}' + ) @staticmethod def record_process_exit(): @@ -184,29 +196,83 @@ def record_process_exit(): returncode = process_returncode if returncode is None: returncode = '"MISSING EXIT CODE"' - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"exit","time":' + ToolchainProfiler.timestamp() + ',"returncode":' + str(returncode) + '}\n]\n') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"exit","time":' + + ToolchainProfiler.timestamp() + + ',"returncode":' + + str(returncode) + + '}\n]\n' + ) @staticmethod def record_subprocess_spawn(process_pid, process_cmdline): expanded_cmdline = response_file.substitute_response_files(process_cmdline) with ToolchainProfiler.log_access() as f: - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"spawn","targetPid":' + str(process_pid) + ',"time":' + ToolchainProfiler.timestamp() + ',"cmdLine":["' + '","'.join(ToolchainProfiler.escape_args(expanded_cmdline)) + '"]}') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"spawn","targetPid":' + + str(process_pid) + + ',"time":' + + ToolchainProfiler.timestamp() + + ',"cmdLine":["' + + '","'.join(ToolchainProfiler.escape_args(expanded_cmdline)) + + '"]}' + ) @staticmethod def record_subprocess_wait(process_pid): with ToolchainProfiler.log_access() as f: - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"wait","targetPid":' + str(process_pid) + ',"time":' + ToolchainProfiler.timestamp() + '}') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"wait","targetPid":' + + str(process_pid) + + ',"time":' + + ToolchainProfiler.timestamp() + + '}' + ) @staticmethod def record_subprocess_finish(process_pid, returncode): with ToolchainProfiler.log_access() as f: - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"finish","targetPid":' + str(process_pid) + ',"time":' + ToolchainProfiler.timestamp() + ',"returncode":' + str(returncode) + '}') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"finish","targetPid":' + + str(process_pid) + + ',"time":' + + ToolchainProfiler.timestamp() + + ',"returncode":' + + str(returncode) + + '}' + ) @staticmethod def enter_block(block_name): with ToolchainProfiler.log_access() as f: - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"enterBlock","name":"' + block_name + '","time":' + ToolchainProfiler.timestamp() + '}') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"enterBlock","name":"' + + block_name + + '","time":' + + ToolchainProfiler.timestamp() + + '}' + ) ToolchainProfiler.block_stack.append(block_name) @@ -222,7 +288,17 @@ def remove_last_occurrence_if_exists(lst, item): def exit_block(block_name): if ToolchainProfiler.remove_last_occurrence_if_exists(ToolchainProfiler.block_stack, block_name): with ToolchainProfiler.log_access() as f: - f.write(',\n{"pid":' + ToolchainProfiler.mypid_str + ',"subprocessPid":' + str(os.getpid()) + ',"op":"exitBlock","name":"' + block_name + '","time":' + ToolchainProfiler.timestamp() + '}') + f.write( + ',\n{"pid":' + + ToolchainProfiler.mypid_str + + ',"subprocessPid":' + + str(os.getpid()) + + ',"op":"exitBlock","name":"' + + block_name + + '","time":' + + ToolchainProfiler.timestamp() + + '}' + ) @staticmethod def exit_all_blocks(): @@ -255,6 +331,7 @@ def imaginary_pid(): ToolchainProfiler.record_process_start() else: + class ToolchainProfiler: @staticmethod def enter_block(block_name): diff --git a/tools/utils.py b/tools/utils.py index c9227b6745b3a..ed65b92df2423 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -43,7 +43,7 @@ def safe_ensure_dirs(dirname): # TODO(sbc): Replace with str.removeprefix once we update to python3.9 def removeprefix(string, prefix): if string.startswith(prefix): - return string[len(prefix):] + return string[len(prefix) :] return string diff --git a/tools/wasm-sourcemap.py b/tools/wasm-sourcemap.py index 15a8707231e6b..3edab7e0d5226 100755 --- a/tools/wasm-sourcemap.py +++ b/tools/wasm-sourcemap.py @@ -33,9 +33,19 @@ def parse_args(): parser = argparse.ArgumentParser(prog='wasm-sourcemap.py', description=__doc__) parser.add_argument('wasm', help='wasm file') parser.add_argument('-o', '--output', help='output source map') - parser.add_argument('-p', '--prefix', nargs='*', help='replace source debug filename prefix for source map', default=[]) - parser.add_argument('-s', '--sources', action='store_true', help='read and embed source files from file system into source map') - parser.add_argument('-l', '--load-prefix', nargs='*', help='replace source debug filename prefix for reading sources from file system (see also --sources)', default=[]) + parser.add_argument( + '-p', '--prefix', nargs='*', help='replace source debug filename prefix for source map', default=[] + ) + parser.add_argument( + '-s', '--sources', action='store_true', help='read and embed source files from file system into source map' + ) + parser.add_argument( + '-l', + '--load-prefix', + nargs='*', + help='replace source debug filename prefix for reading sources from file system (see also --sources)', + default=[], + ) parser.add_argument('-w', nargs='?', help='set output wasm file') parser.add_argument('-x', '--strip', action='store_true', help='removes debug and linking sections') parser.add_argument('-u', '--source-map-url', nargs='?', help='specifies sourceMappingURL section contest') @@ -97,11 +107,11 @@ def encode_vlq(n): def read_var_uint(wasm, pos): n = 0 shift = 0 - b = ord(wasm[pos:pos + 1]) + b = ord(wasm[pos : pos + 1]) pos = pos + 1 while b >= 128: n = n | ((b - 128) << shift) - b = ord(wasm[pos:pos + 1]) + b = ord(wasm[pos : pos + 1]) pos = pos + 1 shift += 7 return n + (b << shift), pos @@ -121,7 +131,12 @@ def strip_debug_sections(wasm): name_len, name_pos = read_var_uint(wasm, section_body) name_end = name_pos + name_len name = wasm[name_pos:name_end] - if name == "linking" or name == "sourceMappingURL" or name.startswith("reloc..debug_") or name.startswith(".debug_"): + if ( + name == "linking" + or name == "sourceMappingURL" + or name.startswith("reloc..debug_") + or name.startswith(".debug_") + ): continue # skip debug related sections stripped = stripped + wasm[section_start:pos] @@ -140,7 +155,9 @@ def encode_uint_var(n): def append_source_mapping(wasm, url): logger.debug('Append sourceMappingURL section') section_name = "sourceMappingURL" - section_content = encode_uint_var(len(section_name)) + section_name.encode() + encode_uint_var(len(url)) + url.encode() + section_content = ( + encode_uint_var(len(section_name)) + section_name.encode() + encode_uint_var(len(url)) + url.encode() + ) return wasm + encode_uint_var(0) + encode_uint_var(len(section_content)) + section_content @@ -168,10 +185,10 @@ def remove_dead_entries(entries): fn_start = entries[block_start]['address'] # Calculate the LEB encoded function size (including size field) fn_size_length = floor(log(entries[cur_entry]['address'] - fn_start + 1, 128)) + 1 - min_live_offset = 1 + fn_size_length # 1 byte is for code section entries + min_live_offset = 1 + fn_size_length # 1 byte is for code section entries if fn_start < min_live_offset: # Remove dead code debug info block. - del entries[block_start:cur_entry + 1] + del entries[block_start : cur_entry + 1] cur_entry = block_start continue cur_entry += 1 @@ -241,7 +258,13 @@ def read_dwarf_entries(wasm, options): files[file.group(1)] = file_path for line in re.finditer(r"\n0x([0-9a-f]+)\s+(\d+)\s+(\d+)\s+(\d+)(.*?end_sequence)?", line_chunk): - entry = {'address': int(line.group(1), 16), 'line': int(line.group(2)), 'column': int(line.group(3)), 'file': files[line.group(4)], 'eos': line.group(5) is not None} + entry = { + 'address': int(line.group(1), 16), + 'line': int(line.group(2)), + 'column': int(line.group(3)), + 'file': files[line.group(4)], + 'eos': line.group(5) is not None, + } if not entry['eos']: entries.append(entry) else: @@ -311,16 +334,20 @@ def build_sourcemap(entries, code_section_offset, prefixes, collect_sources, bas source_id_delta = source_id - last_source_id line_delta = line - last_line column_delta = column - last_column - mappings.append(encode_vlq(address_delta) + encode_vlq(source_id_delta) + encode_vlq(line_delta) + encode_vlq(column_delta)) + mappings.append( + encode_vlq(address_delta) + encode_vlq(source_id_delta) + encode_vlq(line_delta) + encode_vlq(column_delta) + ) last_address = address last_source_id = source_id last_line = line last_column = column - return {'version': 3, - 'sources': sources, - 'sourcesContent': sources_content, - 'names': [], - 'mappings': ','.join(mappings)} + return { + 'version': 3, + 'sources': sources, + 'sourcesContent': sources_content, + 'names': [], + 'mappings': ','.join(mappings), + } def main(): diff --git a/tools/webassembly.py b/tools/webassembly.py index 898d33e35b997..525195f68b454 100644 --- a/tools/webassembly.py +++ b/tools/webassembly.py @@ -3,8 +3,7 @@ # University of Illinois/NCSA Open Source License. Both these licenses can be # found in the LICENSE file. -"""Utilities for manipulating WebAssembly binaries from python. -""" +"""Utilities for manipulating WebAssembly binaries from python.""" from collections import namedtuple from enum import IntEnum @@ -33,9 +32,9 @@ SEG_PASSIVE = 0x1 -PREFIX_MATH = 0xfc -PREFIX_THREADS = 0xfe -PREFIX_SIMD = 0xfd +PREFIX_MATH = 0xFC +PREFIX_THREADS = 0xFE +PREFIX_SIMD = 0xFD SYMBOL_BINDING_MASK = 0x3 SYMBOL_BINDING_GLOBAL = 0x0 @@ -56,7 +55,6 @@ def read_sleb(iobuf): def memoize(method): - @wraps(method) def wrapper(self, *args, **kwargs): assert not kwargs @@ -69,7 +67,6 @@ def wrapper(self, *args, **kwargs): def once(method): - @wraps(method) def helper(self, *args, **kwargs): key = method @@ -80,46 +77,46 @@ def helper(self, *args, **kwargs): class Type(IntEnum): - I32 = 0x7f # -0x1 - I64 = 0x7e # -0x2 - F32 = 0x7d # -0x3 - F64 = 0x7c # -0x4 - V128 = 0x7b # -0x5 - FUNCREF = 0x70 # -0x10 - EXTERNREF = 0x6f # -0x11 - VOID = 0x40 # -0x40 + I32 = 0x7F # -0x1 + I64 = 0x7E # -0x2 + F32 = 0x7D # -0x3 + F64 = 0x7C # -0x4 + V128 = 0x7B # -0x5 + FUNCREF = 0x70 # -0x10 + EXTERNREF = 0x6F # -0x11 + VOID = 0x40 # -0x40 class OpCode(IntEnum): NOP = 0x01 BLOCK = 0x02 - END = 0x0b - BR = 0x0c - BR_TABLE = 0x0e + END = 0x0B + BR = 0x0C + BR_TABLE = 0x0E CALL = 0x10 - DROP = 0x1a + DROP = 0x1A LOCAL_GET = 0x20 LOCAL_SET = 0x21 LOCAL_TEE = 0x22 GLOBAL_GET = 0x23 GLOBAL_SET = 0x24 - RETURN = 0x0f + RETURN = 0x0F I32_CONST = 0x41 I64_CONST = 0x42 F32_CONST = 0x43 F64_CONST = 0x44 - I32_ADD = 0x6a - I64_ADD = 0x7c - REF_NULL = 0xd0 - ATOMIC_PREFIX = 0xfe - MEMORY_PREFIX = 0xfc + I32_ADD = 0x6A + I64_ADD = 0x7C + REF_NULL = 0xD0 + ATOMIC_PREFIX = 0xFE + MEMORY_PREFIX = 0xFC class MemoryOpCode(IntEnum): MEMORY_INIT = 0x08 MEMORY_DROP = 0x09 - MEMORY_COPY = 0x0a - MEMORY_FILL = 0x0b + MEMORY_COPY = 0x0A + MEMORY_FILL = 0x0B class AtomicOpCode(IntEnum): @@ -163,8 +160,8 @@ class DylinkType(IntEnum): class TargetFeaturePrefix(IntEnum): - USED = 0x2b - DISALLOWED = 0x2d + USED = 0x2B + DISALLOWED = 0x2D class InvalidWasmError(BaseException): @@ -176,7 +173,9 @@ class InvalidWasmError(BaseException): Import = namedtuple('Import', ['kind', 'module', 'field', 'type']) Export = namedtuple('Export', ['name', 'kind', 'index']) Global = namedtuple('Global', ['type', 'mutable', 'init']) -Dylink = namedtuple('Dylink', ['mem_size', 'mem_align', 'table_size', 'table_align', 'needed', 'export_info', 'import_info']) +Dylink = namedtuple( + 'Dylink', ['mem_size', 'mem_align', 'table_size', 'table_align', 'needed', 'export_info', 'import_info'] +) Table = namedtuple('Table', ['elem_type', 'limits']) FunctionBody = namedtuple('FunctionBody', ['offset', 'size']) DataSegment = namedtuple('DataSegment', ['flags', 'init', 'offset', 'size']) @@ -186,8 +185,9 @@ class InvalidWasmError(BaseException): class Module: """Extremely minimal wasm module reader. Currently only used for parsing the dylink section.""" + def __init__(self, filename): - self.buf = None # Set this before FS calls below in case they throw. + self.buf = None # Set this before FS calls below in case they throw. self.filename = filename self.size = os.path.getsize(filename) self.buf = open(filename, 'rb') @@ -486,7 +486,7 @@ def get_segments(self): num_segments = self.read_uleb() for _ in range(num_segments): flags = self.read_uleb() - if (flags & SEG_PASSIVE): + if flags & SEG_PASSIVE: init = None else: init = self.read_init() @@ -564,7 +564,7 @@ def get_target_features(self): self.seek(section.offset) assert self.read_string() == 'target_features' features = {} - self.read_byte() # ignore feature count + self.read_byte() # ignore feature count while self.tell() < section.offset + section.size: prefix = TargetFeaturePrefix(self.read_byte()) feature = self.read_string() diff --git a/tools/webidl_binder.py b/tools/webidl_binder.py index 2e54f792d58f8..e8ca2563bcecb 100644 --- a/tools/webidl_binder.py +++ b/tools/webidl_binder.py @@ -57,8 +57,7 @@ def getExtendedAttribute(self, _name): parser = argparse.ArgumentParser() -parser.add_argument('--wasm64', action='store_true', default=False, - help='Build for wasm64') +parser.add_argument('--wasm64', action='store_true', default=False, help='Build for wasm64') parser.add_argument('infile') parser.add_argument('outfile') options = parser.parse_args() @@ -72,10 +71,13 @@ def getExtendedAttribute(self, _name): utils.delete_file(js_output) p = WebIDL.Parser() -p.parse(''' +p.parse( + ''' interface VoidPtr { }; -''' + utils.read_file(input_file)) +''' + + utils.read_file(input_file) +) data = p.finish() interfaces = {} @@ -93,14 +95,17 @@ def getExtendedAttribute(self, _name): # print interfaces # print implements -pre_c = [''' +pre_c = [ + ''' #include #include EM_JS_DEPS(webidl_binder, "$intArrayFromString,$UTF8ToString,$alignMemory"); -'''] +''' +] -mid_c = [''' +mid_c = [ + ''' extern "C" { // Define custom allocator functions that we can force export using @@ -109,30 +114,36 @@ def getExtendedAttribute(self, _name): EMSCRIPTEN_KEEPALIVE void webidl_free(void* p) { free(p); } EMSCRIPTEN_KEEPALIVE void* webidl_malloc(size_t len) { return malloc(len); } -'''] +''' +] def build_constructor(name): implementing_name = implements[name][0] if implements.get(name) else 'WrapperObject' - return [r'''{name}.prototype = Object.create({implementing}.prototype); + return [ + r'''{name}.prototype = Object.create({implementing}.prototype); {name}.prototype.constructor = {name}; {name}.prototype.__class__ = {name}; {name}.__cache__ = {{}}; Module['{name}'] = {name}; -'''.format(name=name, implementing=implementing_name)] +'''.format(name=name, implementing=implementing_name) + ] -mid_js = [''' +mid_js = [ + ''' // Bindings utilities /** @suppress {duplicate} (TODO: avoid emitting this multiple times, it is redundant) */ function WrapperObject() { } -'''] +''' +] mid_js += build_constructor('WrapperObject') -mid_js += [''' +mid_js += [ + ''' /** @suppress {duplicate} (TODO: avoid emitting this multiple times, it is redundant) @param {*=} __class__ */ function getCache(__class__) { @@ -305,7 +316,8 @@ def build_constructor(name): } return value; } -'''] +''' +] C_FLOATS = ['float', 'double'] @@ -386,10 +398,22 @@ def type_to_cdec(raw): return ret + '*' -def render_function(class_name, func_name, sigs, return_type, non_pointer, # noqa: C901, PLR0912, PLR0915 - copy, operator, constructor, is_static, func_scope, - call_content=None, const=False, array_attribute=False, - bind_to=None): +def render_function( # noqa: C901, PLR0912, PLR0915 + class_name, + func_name, + sigs, + return_type, + non_pointer, + copy, + operator, + constructor, + is_static, + func_scope, + call_content=None, + const=False, + array_attribute=False, + bind_to=None, +): """Future modifications should consider refactoring to reduce complexity. * The McCabe cyclomatiic complexity is currently 67 vs 10 recommended. @@ -443,7 +467,10 @@ def render_function(class_name, func_name, sigs, return_type, non_pointer, # no if options.wasm64 and ptr_rtn: call_prefix += 'Number(' - args = [(all_args[i].identifier.name if isinstance(all_args[i], WebIDL.IDLArgument) else ('arg%d' % i)) for i in range(max_args)] + args = [ + (all_args[i].identifier.name if isinstance(all_args[i], WebIDL.IDLArgument) else ('arg%d' % i)) + for i in range(max_args) + ] if not constructor and not is_static: body = ' var self = this.ptr;\n' if options.wasm64: @@ -459,7 +486,7 @@ def render_function(class_name, func_name, sigs, return_type, non_pointer, # no def is_ptr_arg(i): t = all_args[i].type - return (t.isArray() or t.isAny() or t.isString() or t.isObject() or t.isInterface()) + return t.isArray() or t.isAny() or t.isString() or t.isObject() or t.isInterface() for i, (js_arg, arg) in enumerate(zip(args, all_args)): optional = i >= min_args @@ -489,23 +516,33 @@ def is_ptr_arg(i): if arg.type.isNumeric(): if arg.type.isInteger(): if all_checks: - body += " assert(typeof {0} === 'number' && !isNaN({0}), '{1}Expecting ');\n".format(js_arg, check_msg) + body += " assert(typeof {0} === 'number' && !isNaN({0}), '{1}Expecting ');\n".format( + js_arg, check_msg + ) else: if all_checks: body += " assert(typeof {0} === 'number', '{1}Expecting ');\n".format(js_arg, check_msg) # No transform needed for numbers elif arg.type.isBoolean(): if all_checks: - body += " assert(typeof {0} === 'boolean' || (typeof {0} === 'number' && !isNaN({0})), '{1}Expecting ');\n".format(js_arg, check_msg) + body += " assert(typeof {0} === 'boolean' || (typeof {0} === 'number' && !isNaN({0})), '{1}Expecting ');\n".format( + js_arg, check_msg + ) # No transform needed for booleans elif arg.type.isString(): # Strings can be DOM strings or pointers. if all_checks: - body += " assert(typeof {0} === 'string' || ({0} && typeof {0} === 'object' && typeof {0}.ptr === 'number'), '{1}Expecting ');\n".format(js_arg, check_msg) - do_default = True # legacy path is fast enough for strings. + body += " assert(typeof {0} === 'string' || ({0} && typeof {0} === 'object' && typeof {0}.ptr === 'number'), '{1}Expecting ');\n".format( + js_arg, check_msg + ) + do_default = True # legacy path is fast enough for strings. elif arg.type.isInterface(): if all_checks: - body += " assert(typeof {0} === 'object' && typeof {0}.ptr === 'number', '{1}Expecting ');\n".format(js_arg, check_msg) + body += ( + " assert(typeof {0} === 'object' && typeof {0}.ptr === 'number', '{1}Expecting ');\n".format( + js_arg, check_msg + ) + ) if optional: body += " if(typeof {0} !== 'undefined' && {0} !== null) {{ {0} = {0}.ptr }};\n".format(js_arg) else: @@ -561,9 +598,14 @@ def make_call_args(i): else: after_call = '; ' + cache + 'return' args_for_call = make_call_args(i) - body += ' if (%s === undefined) { %s_%s(%s)%s%s }\n' % (args[i], call_prefix, c_names[i], - args_for_call, - call_postfix, after_call) + body += ' if (%s === undefined) { %s_%s(%s)%s%s }\n' % ( + args[i], + call_prefix, + c_names[i], + args_for_call, + call_postfix, + after_call, + ) dbg(call_prefix) c_names[max_args] = f'emscripten_bind_{bindings_name}_{max_args}' args_for_call = make_call_args(len(args)) @@ -575,10 +617,13 @@ def make_call_args(i): declare_name = ' ' + func_name else: declare_name = '' - mid_js.append(r'''function%s(%s) { + mid_js.append( + r'''function%s(%s) { %s }; -''' % (declare_name, ', '.join(args), body[:-1])) +''' + % (declare_name, ', '.join(args), body[:-1]) + ) # C @@ -622,9 +667,38 @@ def make_call_args(i): cast_self = 'dynamic_cast<' + type_to_c(func_scope) + '>(' + cast_self + ')' maybe_deref = deref_if_nonpointer(raw[0]) operator = operator.strip() - if operator in ["+", "-", "*", "/", "%", "^", "&", "|", "=", - "<", ">", "+=", "-=", "*=", "/=", "%=", "^=", "&=", "|=", "<<", ">>", ">>=", - "<<=", "==", "!=", "<=", ">=", "<=>", "&&", "||"]: + if operator in [ + "+", + "-", + "*", + "/", + "%", + "^", + "&", + "|", + "=", + "<", + ">", + "+=", + "-=", + "*=", + "/=", + "%=", + "^=", + "&=", + "|=", + "<<", + ">>", + ">>=", + "<<=", + "==", + "!=", + "<=", + ">=", + "<=>", + "&&", + "||", + ]: call = '(*%s %s %s%s)' % (cast_self, operator, maybe_deref, args[0]) elif operator == '[]': call = '((*%s)[%s%s])' % (cast_self, maybe_deref, args[0]) @@ -646,33 +720,64 @@ def make_call_args(i): c_return_type = type_to_c(return_type) maybe_const = 'const ' if const else '' - mid_c.append(r''' + mid_c.append( + r''' %s%s EMSCRIPTEN_KEEPALIVE %s(%s) { %s %s%s%s; } -''' % (maybe_const, type_to_c(class_name) if constructor else c_return_type, c_names[i], full_args, pre, return_prefix, call, return_postfix)) +''' + % ( + maybe_const, + type_to_c(class_name) if constructor else c_return_type, + c_names[i], + full_args, + pre, + return_prefix, + call, + return_postfix, + ) + ) if not constructor: if i == max_args: dec_args = ', '.join([type_to_cdec(raw[j]) + ' ' + args[j] for j in range(i)]) - js_call_args = ', '.join(['%s%s' % (('(ptrdiff_t)' if sig[j] in interfaces else '') + take_addr_if_nonpointer(raw[j]), args[j]) for j in range(i)]) - em_asm_macro = 'EM_ASM_%s' % ('PTR' if c_return_type[-1] == '*' else 'INT' if c_return_type not in C_FLOATS else 'DOUBLE') - - js_impl_methods.append(r''' %s %s(%s) %s { + js_call_args = ', '.join( + [ + '%s%s' % (('(ptrdiff_t)' if sig[j] in interfaces else '') + take_addr_if_nonpointer(raw[j]), args[j]) + for j in range(i) + ] + ) + em_asm_macro = 'EM_ASM_%s' % ( + 'PTR' if c_return_type[-1] == '*' else 'INT' if c_return_type not in C_FLOATS else 'DOUBLE' + ) + + js_impl_methods.append( + r''' %s %s(%s) %s { %s (%s) %s({ var self = Module['getCache'](Module['%s'])[$0]; if (!self.hasOwnProperty('%s')) throw 'a JSImplementation must implement all functions, you forgot %s::%s.'; %sself['%s'](%s)%s; }, (ptrdiff_t)this%s); - }''' % (c_return_type, func_name, dec_args, maybe_const, - basic_return, c_return_type, em_asm_macro, - class_name, - func_name, class_name, func_name, - return_prefix, - func_name, - ','.join(['$%d' % i for i in range(1, max_args + 1)]), - return_postfix, - (', ' if js_call_args else '') + js_call_args)) + }''' + % ( + c_return_type, + func_name, + dec_args, + maybe_const, + basic_return, + c_return_type, + em_asm_macro, + class_name, + func_name, + class_name, + func_name, + return_prefix, + func_name, + ','.join(['$%d' % i for i in range(1, max_args + 1)]), + return_postfix, + (', ' if js_call_args else '') + js_call_args, + ) + ) def add_bounds_check_impl(): @@ -740,7 +845,10 @@ def add_bounds_check_impl(): # Ensure a constructor even if one is not specified. if not any(m.identifier.name == name for m in interface.members): - mid_js += ['%s\nfunction %s() { throw "cannot construct a %s, no constructor in IDL" }\n' % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, name)] + mid_js += [ + '%s\nfunction %s() { throw "cannot construct a %s, no constructor in IDL" }\n' + % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, name) + ] mid_js += build_constructor(name) for m in interface.members: @@ -768,18 +876,24 @@ def add_bounds_check_impl(): assert return_type == ret.name, 'overloads must have the same return type' for i in range(len(args) + 1): if i == len(args) or args[i].optional: - assert i not in sigs, 'overloading must differentiate by # of arguments (cannot have two signatures that differ by types but not by length)' + assert ( + i not in sigs + ), 'overloading must differentiate by # of arguments (cannot have two signatures that differ by types but not by length)' sigs[i] = args[:i] - render_function(name, - m.identifier.name, sigs, return_type, - m.getExtendedAttribute('Ref'), - m.getExtendedAttribute('Value'), - (m.getExtendedAttribute('Operator') or [None])[0], - constructor, - is_static=m.isStatic(), - func_scope=m.parentScope.identifier.name, - const=m.getExtendedAttribute('Const'), - bind_to=(m.getExtendedAttribute('BindTo') or [None])[0]) + render_function( + name, + m.identifier.name, + sigs, + return_type, + m.getExtendedAttribute('Ref'), + m.getExtendedAttribute('Value'), + (m.getExtendedAttribute('Operator') or [None])[0], + constructor, + is_static=m.isStatic(), + func_scope=m.parentScope.identifier.name, + const=m.getExtendedAttribute('Const'), + bind_to=(m.getExtendedAttribute('BindTo') or [None])[0], + ) mid_js += ['\n'] if constructor: mid_js += build_constructor(name) @@ -791,12 +905,10 @@ def add_bounds_check_impl(): if m.type.isArray(): get_sigs = {1: [Dummy(type=WebIDL.BuiltinTypes[WebIDL.IDLBuiltinType.Types.long])]} - set_sigs = {2: [Dummy(type=WebIDL.BuiltinTypes[WebIDL.IDLBuiltinType.Types.long]), - Dummy(type=m.type.inner)]} + set_sigs = {2: [Dummy(type=WebIDL.BuiltinTypes[WebIDL.IDLBuiltinType.Types.long]), Dummy(type=m.type.inner)]} get_call_content = take_addr_if_nonpointer(m) + 'self->' + attr + '[arg0]' set_call_content = 'self->' + attr + '[arg0] = ' + deref_if_nonpointer(m) + 'arg1' if m.getExtendedAttribute('BoundsChecked'): - bounds_check = "array_bounds_check(sizeof(self->%s) / sizeof(self->%s[0]), arg0)" % (attr, attr) add_bounds_check_impl() @@ -809,69 +921,99 @@ def add_bounds_check_impl(): set_call_content = 'self->' + attr + ' = ' + deref_if_nonpointer(m) + 'arg0' get_name = 'get_' + attr - mid_js += [r'''%s -%s.prototype['%s'] = %s.prototype.%s = ''' % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, get_name, name, get_name)] - render_function(name, - get_name, get_sigs, m.type.name, - None, - None, - None, - False, - False, - func_scope=interface, - call_content=get_call_content, - const=m.getExtendedAttribute('Const'), - array_attribute=m.type.isArray()) + mid_js += [ + r'''%s +%s.prototype['%s'] = %s.prototype.%s = ''' + % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, get_name, name, get_name) + ] + render_function( + name, + get_name, + get_sigs, + m.type.name, + None, + None, + None, + False, + False, + func_scope=interface, + call_content=get_call_content, + const=m.getExtendedAttribute('Const'), + array_attribute=m.type.isArray(), + ) if m.readonly: - mid_js += [r''' + mid_js += [ + r''' /** @suppress {checkTypes} */ Object.defineProperty(%s.prototype, '%s', { get: %s.prototype.%s }); -''' % (name, attr, name, get_name)] +''' + % (name, attr, name, get_name) + ] else: set_name = 'set_' + attr - mid_js += [r''' + mid_js += [ + r''' %s -%s.prototype['%s'] = %s.prototype.%s = ''' % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, set_name, name, set_name)] - render_function(name, - set_name, set_sigs, 'Void', - None, - None, - None, - False, - False, - func_scope=interface, - call_content=set_call_content, - const=m.getExtendedAttribute('Const'), - array_attribute=m.type.isArray()) - mid_js += [r''' +%s.prototype['%s'] = %s.prototype.%s = ''' + % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, set_name, name, set_name) + ] + render_function( + name, + set_name, + set_sigs, + 'Void', + None, + None, + None, + False, + False, + func_scope=interface, + call_content=set_call_content, + const=m.getExtendedAttribute('Const'), + array_attribute=m.type.isArray(), + ) + mid_js += [ + r''' /** @suppress {checkTypes} */ Object.defineProperty(%s.prototype, '%s', { get: %s.prototype.%s, set: %s.prototype.%s }); -''' % (name, attr, name, get_name, name, set_name)] +''' + % (name, attr, name, get_name, name, set_name) + ] if not interface.getExtendedAttribute('NoDelete'): - mid_js += [r''' + mid_js += [ + r''' %s -%s.prototype['__destroy__'] = %s.prototype.__destroy__ = ''' % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, name)] - render_function(name, - '__destroy__', {0: []}, 'Void', - None, - None, - None, - False, - False, - func_scope=interface, - call_content='delete self') +%s.prototype['__destroy__'] = %s.prototype.__destroy__ = ''' + % (CONSTRUCTOR_CLOSURE_SUPPRESSIONS, name, name) + ] + render_function( + name, + '__destroy__', + {0: []}, + 'Void', + None, + None, + None, + False, + False, + func_scope=interface, + call_content='delete self', + ) # Emit C++ class implementation that calls into JS implementation if js_impl: - pre_c += [''' + pre_c += [ + ''' class %s : public %s { public: %s }; -''' % (name, type_to_c(js_impl, non_pointing=True), '\n'.join(js_impl_methods))] +''' + % (name, type_to_c(js_impl, non_pointing=True), '\n'.join(js_impl_methods)) + ] deferred_js = [] @@ -881,10 +1023,13 @@ class %s : public %s { for value in enum.values(): function_id = '%s_%s' % (name, value.split('::')[-1]) function_id = 'emscripten_enum_%s' % function_id - mid_c += ['''%s EMSCRIPTEN_KEEPALIVE %s() { + mid_c += [ + '''%s EMSCRIPTEN_KEEPALIVE %s() { return %s; } -''' % (name, function_id, value)] +''' + % (name, function_id, value) + ] symbols = value.split('::') if len(symbols) == 1: identifier = symbols[0] @@ -902,7 +1047,8 @@ class %s : public %s { mid_c += ['\n}\n\n'] if len(deferred_js): - mid_js += [''' + mid_js += [ + ''' (function() { function setupEnums() { %s @@ -910,7 +1056,9 @@ class %s : public %s { if (runtimeInitialized) setupEnums(); else addOnInit(setupEnums); })(); -''' % '\n '.join(deferred_js)] +''' + % '\n '.join(deferred_js) + ] # Write